Merge pull request #53037 from smarterclayton/verify_client_cert

cmd/kubelet/app/server.go

@@ -342,7 +342,9 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) {
 				if err != nil {
 					return err
 				}
-				if err := kubeletcertificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager); err != nil {
+				// we set exitIfExpired to true because we use this client configuration to request new certs - if we are unable
+				// to request new certs, we will be unable to continue normal operation
+				if err := kubeletcertificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager, true); err != nil {
 					return err
 				}
 			}

pkg/kubelet/certificate/bootstrap/BUILD

@@ -22,14 +22,16 @@ go_library(
     srcs = ["bootstrap.go"],
     importpath = "k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap",
     deps = [
-        "//pkg/kubelet/util/csr:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
+        "//vendor/k8s.io/client-go/transport:go_default_library",
         "//vendor/k8s.io/client-go/util/cert:go_default_library",
+        "//vendor/k8s.io/client-go/util/certificate/csr:go_default_library",
     ],
 )
 

pkg/kubelet/certificate/bootstrap/bootstrap.go

@@ -20,16 +20,19 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/golang/glog"
 
 	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	certificates "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/client-go/transport"
 	certutil "k8s.io/client-go/util/cert"
-	"k8s.io/kubernetes/pkg/kubelet/util/csr"
+	"k8s.io/client-go/util/certificate/csr"
 )
 
 const (
@@ -42,17 +45,15 @@ const (
 // On success, a kubeconfig file referencing the generated key and obtained certificate is written to kubeconfigPath.
 // The certificate and key file are stored in certDir.
 func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string, nodeName types.NodeName) error {
-	// Short-circuit if the kubeconfig file already exists.
-	// TODO: inspect the kubeconfig, ensure a rest client can be built from it, verify client cert expiration, etc.
-	_, err := os.Stat(kubeconfigPath)
-	if err == nil {
-		glog.V(2).Infof("Kubeconfig %s exists, skipping bootstrap", kubeconfigPath)
-		return nil
-	}
-	if !os.IsNotExist(err) {
-		glog.Errorf("Error reading kubeconfig %s, skipping bootstrap: %v", kubeconfigPath, err)
+	// Short-circuit if the kubeconfig file exists and is valid.
+	ok, err := verifyBootstrapClientConfig(kubeconfigPath)
+	if err != nil {
 		return err
 	}
+	if ok {
+		glog.V(2).Infof("Kubeconfig %s exists and is valid, skipping bootstrap", kubeconfigPath)
+		return nil
+	}
 
 	glog.V(2).Info("Using bootstrap kubeconfig to generate TLS client cert, key and kubeconfig file")
 
@@ -72,6 +73,17 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string,
 	if err != nil {
 		return fmt.Errorf("unable to build bootstrap key path: %v", err)
 	}
+	// If we are unable to generate a CSR, we remove our key file and start fresh.
+	// This method is used before enabling client rotation and so we must ensure we
+	// can make forward progress if we crash and exit when a CSR exists but the cert
+	// it is signed for has expired.
+	defer func() {
+		if !success {
+			if err := os.Remove(keyPath); err != nil && !os.IsNotExist(err) {
+				glog.Warningf("Cannot clean up the key file %q: %v", keyPath, err)
+			}
+		}
+	}()
 	keyData, _, err := certutil.LoadOrGenerateKeyFile(keyPath)
 	if err != nil {
 		return err
@@ -82,6 +94,13 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string,
 	if err != nil {
 		return fmt.Errorf("unable to build bootstrap client cert path: %v", err)
 	}
+	defer func() {
+		if !success {
+			if err := os.Remove(certPath); err != nil && !os.IsNotExist(err) {
+				glog.Warningf("Cannot clean up the cert file %q: %v", certPath, err)
+			}
+		}
+	}()
 	certData, err := csr.RequestNodeCertificate(bootstrapClient.CertificateSigningRequests(), keyData, nodeName)
 	if err != nil {
 		return err
@@ -89,13 +108,6 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string,
 	if err := certutil.WriteCert(certPath, certData); err != nil {
 		return err
 	}
-	defer func() {
-		if !success {
-			if err := os.Remove(certPath); err != nil {
-				glog.Warningf("Cannot clean up the cert file %q: %v", certPath, err)
-			}
-		}
-	}()
 
 	// Get the CA data from the bootstrap client config.
 	caFile, caData := bootstrapClientConfig.CAFile, []byte{}
@@ -150,3 +162,48 @@ func loadRESTClientConfig(kubeconfig string) (*restclient.Config, error) {
 		loader,
 	).ClientConfig()
 }
+
+// verifyBootstrapClientConfig checks the provided kubeconfig to see if it has a valid
+// client certificate. It returns true if the kubeconfig is valid, or an error if bootstrapping
+// should stop immediately.
+func verifyBootstrapClientConfig(kubeconfigPath string) (bool, error) {
+	_, err := os.Stat(kubeconfigPath)
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("error reading existing bootstrap kubeconfig %s: %v", kubeconfigPath, err)
+	}
+	bootstrapClientConfig, err := loadRESTClientConfig(kubeconfigPath)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("Unable to read existing bootstrap client config: %v", err))
+		return false, nil
+	}
+	transportConfig, err := bootstrapClientConfig.TransportConfig()
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("Unable to load transport configuration from existing bootstrap client config: %v", err))
+		return false, nil
+	}
+	// has side effect of populating transport config data fields
+	if _, err := transport.TLSConfigFor(transportConfig); err != nil {
+		utilruntime.HandleError(fmt.Errorf("Unable to load TLS configuration from existing bootstrap client config: %v", err))
+		return false, nil
+	}
+	certs, err := certutil.ParseCertsPEM(transportConfig.TLS.CertData)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("Unable to load TLS certificates from existing bootstrap client config: %v", err))
+		return false, nil
+	}
+	if len(certs) == 0 {
+		utilruntime.HandleError(fmt.Errorf("Unable to read TLS certificates from existing bootstrap client config: %v", err))
+		return false, nil
+	}
+	now := time.Now()
+	for _, cert := range certs {
+		if now.After(cert.NotAfter) {
+			utilruntime.HandleError(fmt.Errorf("Part of the existing bootstrap client certificate is expired: %s", cert.NotAfter))
+			return false, nil
+		}
+	}
+	return true, nil
+}

pkg/kubelet/certificate/transport.go

@@ -45,13 +45,13 @@ import (
 //
 // stopCh should be used to indicate when the transport is unused and doesn't need
 // to continue checking the manager.
-func UpdateTransport(stopCh <-chan struct{}, clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error {
-	return updateTransport(stopCh, 10*time.Second, clientConfig, clientCertificateManager)
+func UpdateTransport(stopCh <-chan struct{}, clientConfig *restclient.Config, clientCertificateManager certificate.Manager, exitIfExpired bool) error {
+	return updateTransport(stopCh, 10*time.Second, clientConfig, clientCertificateManager, exitIfExpired)
 }
 
 // updateTransport is an internal method that exposes how often this method checks that the
 // client cert has changed. Intended for testing.
-func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error {
+func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, clientCertificateManager certificate.Manager, exitIfExpired bool) error {
 	if clientConfig.Transport != nil {
 		return fmt.Errorf("there is already a transport configured")
 	}
@@ -80,6 +80,13 @@ func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig
 	lastCert := clientCertificateManager.Current()
 	go wait.Until(func() {
 		curr := clientCertificateManager.Current()
+		if exitIfExpired && curr != nil && time.Now().After(curr.Leaf.NotAfter) {
+			if clientCertificateManager.ServerHealthy() {
+				glog.Fatalf("The currently active client certificate has expired and the server is responsive, exiting.")
+			} else {
+				glog.Errorf("The currently active client certificate has expired, but the server is not responsive. A restart may be necessary to retrieve new initial credentials.")
+			}
+		}
 		if curr == nil || lastCert == curr {
 			// Cert hasn't been rotated.
 			return

pkg/kubelet/certificate/transport_test.go

@@ -114,13 +114,16 @@ func newCertificateData(certificatePEM string, keyPEM string) *certificateData {
 }
 
 type fakeManager struct {
-	cert atomic.Value // Always a *tls.Certificate
+	cert    atomic.Value // Always a *tls.Certificate
+	healthy bool
 }
 
 func (f *fakeManager) SetCertificateSigningRequestClient(certificatesclient.CertificateSigningRequestInterface) error {
 	return nil
 }
 
+func (f *fakeManager) ServerHealthy() bool { return f.healthy }
+
 func (f *fakeManager) Start() {}
 
 func (f *fakeManager) Current() *tls.Certificate {
@@ -184,7 +187,7 @@ func TestRotateShutsDownConnections(t *testing.T) {
 	}
 
 	// Check for a new cert every 10 milliseconds
-	if err := updateTransport(stop, 10*time.Millisecond, c, m); err != nil {
+	if err := updateTransport(stop, 10*time.Millisecond, c, m, false); err != nil {
 		t.Fatal(err)
 	}
 

pkg/kubelet/util/BUILD

@@ -53,7 +53,6 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//pkg/kubelet/util/cache:all-srcs",
-        "//pkg/kubelet/util/csr:all-srcs",
         "//pkg/kubelet/util/format:all-srcs",
         "//pkg/kubelet/util/ioutils:all-srcs",
         "//pkg/kubelet/util/queue:all-srcs",

staging/src/k8s.io/client-go/tools/cache/listwatch.go

@@ -25,6 +25,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/pager"
@@ -103,6 +104,8 @@ func (lw *ListWatch) Watch(options metav1.ListOptions) (watch.Interface, error)
 	return lw.WatchFunc(options)
 }
 
+// ListWatchUntil checks the provided conditions against the items returned by the list watcher, returning wait.ErrWaitTimeout
+// if timeout is exceeded without all conditions returning true, or an error if an error occurs.
 // TODO: check for watch expired error and retry watch from latest point?  Same issue exists for Until.
 func ListWatchUntil(timeout time.Duration, lw ListerWatcher, conditions ...watch.ConditionFunc) (*watch.Event, error) {
 	if len(conditions) == 0 {
@@ -166,5 +169,10 @@ func ListWatchUntil(timeout time.Duration, lw ListerWatcher, conditions ...watch
 		return nil, err
 	}
 
-	return watch.Until(timeout, watchInterface, remainingConditions...)
+	evt, err := watch.Until(timeout, watchInterface, remainingConditions...)
+	if err == watch.ErrWatchClosed {
+		// present a consistent error interface to callers
+		err = wait.ErrWaitTimeout
+	}
+	return evt, err
 }

staging/src/k8s.io/client-go/util/certificate/BUILD

@@ -19,7 +19,10 @@ go_test(
     tags = ["automanaged"],
     deps = [
         "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
         "//vendor/k8s.io/client-go/util/cert:go_default_library",
@@ -37,12 +40,12 @@ go_library(
     deps = [
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
         "//vendor/k8s.io/client-go/util/cert:go_default_library",
+        "//vendor/k8s.io/client-go/util/certificate/csr:go_default_library",
     ],
 )
 
@@ -55,7 +58,10 @@ filegroup(
 
 filegroup(
     name = "all-srcs",
-    srcs = [":package-srcs"],
+    srcs = [
+        ":package-srcs",
+        "//staging/src/k8s.io/client-go/util/certificate/csr:all-srcs",
+    ],
     tags = ["automanaged"],
     visibility = ["//visibility:public"],
 )

staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go

@@ -27,7 +27,10 @@ import (
 	"time"
 
 	certificates "k8s.io/api/certificates/v1beta1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
 	watch "k8s.io/apimachinery/pkg/watch"
 	certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 )
@@ -288,6 +291,7 @@ func TestRotateCertWaitingForResultError(t *testing.T) {
 		},
 	}
 
+	certificateWaitBackoff = wait.Backoff{Steps: 1}
 	if success, err := m.rotateCerts(); success {
 		t.Errorf("Got success from 'rotateCerts', wanted failure.")
 	} else if err != nil {
@@ -612,10 +616,14 @@ func TestInitializeOtherRESTClients(t *testing.T) {
 			} else {
 				m.setRotationDeadline()
 				if m.shouldRotate() {
-					if success, err := certificateManager.(*manager).rotateCerts(); !success {
-						t.Errorf("Got failure from 'rotateCerts', expected success")
-					} else if err != nil {
+					success, err := certificateManager.(*manager).rotateCerts()
+					if err != nil {
 						t.Errorf("Got error %v, expected none.", err)
+						return
+					}
+					if !success {
+						t.Errorf("Unexpected response 'rotateCerts': %t", success)
+						return
 					}
 				}
 			}
@@ -628,6 +636,161 @@ func TestInitializeOtherRESTClients(t *testing.T) {
 	}
 }
 
+func TestServerHealth(t *testing.T) {
+	type certs struct {
+		storeCert               *certificateData
+		bootstrapCert           *certificateData
+		apiCert                 *certificateData
+		expectedCertBeforeStart *certificateData
+		expectedCertAfterStart  *certificateData
+	}
+
+	updatedCerts := certs{
+		storeCert:               storeCertData,
+		bootstrapCert:           bootstrapCertData,
+		apiCert:                 apiServerCertData,
+		expectedCertBeforeStart: storeCertData,
+		expectedCertAfterStart:  apiServerCertData,
+	}
+
+	currentCerts := certs{
+		storeCert:               storeCertData,
+		bootstrapCert:           bootstrapCertData,
+		apiCert:                 apiServerCertData,
+		expectedCertBeforeStart: storeCertData,
+		expectedCertAfterStart:  storeCertData,
+	}
+
+	testCases := []struct {
+		description string
+		certs
+
+		failureType fakeClientFailureType
+		clientErr   error
+
+		expectRotateFail bool
+		expectHealthy    bool
+	}{
+		{
+			description:   "Current certificate, bootstrap certificate",
+			certs:         updatedCerts,
+			expectHealthy: true,
+		},
+		{
+			description: "Generic error on create",
+			certs:       currentCerts,
+
+			failureType:      createError,
+			expectRotateFail: true,
+		},
+		{
+			description: "Unauthorized error on create",
+			certs:       currentCerts,
+
+			failureType:      createError,
+			clientErr:        errors.NewUnauthorized("unauthorized"),
+			expectRotateFail: true,
+			expectHealthy:    true,
+		},
+		{
+			description: "Generic unauthorized error on create",
+			certs:       currentCerts,
+
+			failureType:      createError,
+			clientErr:        errors.NewGenericServerResponse(401, "POST", schema.GroupResource{}, "", "", 0, true),
+			expectRotateFail: true,
+			expectHealthy:    true,
+		},
+		{
+			description: "Generic not found error on create",
+			certs:       currentCerts,
+
+			failureType:      createError,
+			clientErr:        errors.NewGenericServerResponse(404, "POST", schema.GroupResource{}, "", "", 0, true),
+			expectRotateFail: true,
+			expectHealthy:    false,
+		},
+		{
+			description: "Not found error on create",
+			certs:       currentCerts,
+
+			failureType:      createError,
+			clientErr:        errors.NewGenericServerResponse(404, "POST", schema.GroupResource{}, "", "", 0, false),
+			expectRotateFail: true,
+			expectHealthy:    true,
+		},
+		{
+			description: "Conflict error on watch",
+			certs:       currentCerts,
+
+			failureType:      watchError,
+			clientErr:        errors.NewGenericServerResponse(409, "POST", schema.GroupResource{}, "", "", 0, false),
+			expectRotateFail: true,
+			expectHealthy:    false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			certificateStore := &fakeStore{
+				cert: tc.storeCert.certificate,
+			}
+
+			certificateManager, err := NewManager(&Config{
+				Template: &x509.CertificateRequest{
+					Subject: pkix.Name{
+						Organization: []string{"system:nodes"},
+						CommonName:   "system:node:fake-node-name",
+					},
+				},
+				Usages: []certificates.KeyUsage{
+					certificates.UsageDigitalSignature,
+					certificates.UsageKeyEncipherment,
+					certificates.UsageClientAuth,
+				},
+				CertificateStore:        certificateStore,
+				BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM,
+				BootstrapKeyPEM:         tc.bootstrapCert.keyPEM,
+				CertificateSigningRequestClient: &fakeClient{
+					certificatePEM: tc.apiCert.certificatePEM,
+					failureType:    tc.failureType,
+					err:            tc.clientErr,
+				},
+			})
+			if err != nil {
+				t.Errorf("Got %v, wanted no error.", err)
+			}
+
+			certificate := certificateManager.Current()
+			if !certificatesEqual(certificate, tc.expectedCertBeforeStart.certificate) {
+				t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertBeforeStart.certificate))
+			}
+
+			if _, ok := certificateManager.(*manager); !ok {
+				t.Errorf("Expected a '*manager' from 'NewManager'")
+			} else {
+				success, err := certificateManager.(*manager).rotateCerts()
+				if err != nil {
+					t.Errorf("Got error %v, expected none.", err)
+					return
+				}
+				if !success != tc.expectRotateFail {
+					t.Errorf("Unexpected response 'rotateCerts': %t", success)
+					return
+				}
+				if actual := certificateManager.(*manager).ServerHealthy(); actual != tc.expectHealthy {
+					t.Errorf("Unexpected manager server health: %t", actual)
+				}
+			}
+
+			certificate = certificateManager.Current()
+			if !certificatesEqual(certificate, tc.expectedCertAfterStart.certificate) {
+				t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertAfterStart.certificate))
+			}
+		})
+	}
+}
+
 type fakeClientFailureType int
 
 const (
@@ -641,10 +804,29 @@ type fakeClient struct {
 	certificatesclient.CertificateSigningRequestInterface
 	failureType    fakeClientFailureType
 	certificatePEM []byte
+	err            error
+}
+
+func (c fakeClient) List(opts v1.ListOptions) (*certificates.CertificateSigningRequestList, error) {
+	if c.failureType == watchError {
+		if c.err != nil {
+			return nil, c.err
+		}
+		return nil, fmt.Errorf("Watch error")
+	}
+	csrReply := certificates.CertificateSigningRequestList{
+		Items: []certificates.CertificateSigningRequest{
+			{ObjectMeta: v1.ObjectMeta{UID: "fake-uid"}},
+		},
+	}
+	return &csrReply, nil
 }
 
 func (c fakeClient) Create(*certificates.CertificateSigningRequest) (*certificates.CertificateSigningRequest, error) {
 	if c.failureType == createError {
+		if c.err != nil {
+			return nil, c.err
+		}
 		return nil, fmt.Errorf("Create error")
 	}
 	csrReply := certificates.CertificateSigningRequest{}
@@ -654,6 +836,9 @@ func (c fakeClient) Create(*certificates.CertificateSigningRequest) (*certificat
 
 func (c fakeClient) Watch(opts v1.ListOptions) (watch.Interface, error) {
 	if c.failureType == watchError {
+		if c.err != nil {
+			return nil, c.err
+		}
 		return nil, fmt.Errorf("Watch error")
 	}
 	return &fakeWatch{

pkg/kubelet/util/csr/BUILD → staging/src/k8s.io/client-go/util/certificate/csr/BUILD

@@ -9,9 +9,8 @@ load(
 go_library(
     name = "go_default_library",
     srcs = ["csr.go"],
-    importpath = "k8s.io/kubernetes/pkg/kubelet/util/csr",
+    importpath = "k8s.io/client-go/util/certificate/csr",
     deps = [
-        "//pkg/apis/certificates/v1beta1:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
@@ -19,6 +18,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
         "//vendor/k8s.io/client-go/tools/cache:go_default_library",
@@ -42,7 +42,7 @@ filegroup(
 go_test(
     name = "go_default_test",
     srcs = ["csr_test.go"],
-    importpath = "k8s.io/kubernetes/pkg/kubelet/util/csr",
+    importpath = "k8s.io/client-go/util/certificate/csr",
     library = ":go_default_library",
     deps = [
         "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",

pkg/kubelet/util/csr/csr.go → staging/src/k8s.io/client-go/util/certificate/csr/csr.go

@@ -19,25 +19,26 @@ package csr
 import (
 	"crypto"
 	"crypto/sha512"
+	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/base64"
+	"encoding/pem"
 	"fmt"
+	"github.com/golang/glog"
 	"reflect"
 	"time"
 
-	"github.com/golang/glog"
-
 	certificates "k8s.io/api/certificates/v1beta1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
 	certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	"k8s.io/client-go/tools/cache"
 	certutil "k8s.io/client-go/util/cert"
-	certhelper "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
 )
 
 // RequestNodeCertificate will create a certificate signing request for a node
@@ -68,16 +69,20 @@ func RequestNodeCertificate(client certificatesclient.CertificateSigningRequestI
 		certificates.UsageClientAuth,
 	}
 	name := digestedName(privateKeyData, subject, usages)
-	return requestCertificate(client, csrData, name, usages, privateKey)
+	req, err := RequestCertificate(client, csrData, name, usages, privateKey)
+	if err != nil {
+		return nil, err
+	}
+	return WaitForCertificate(client, req, 3600*time.Second)
 }
 
-// requestCertificate will either use an existing (if this process has run
+// RequestCertificate will either use an existing (if this process has run
 // before but not to completion) or create a certificate signing request using the
 // PEM encoded CSR and send it to API server, then it will watch the object's
 // status, once approved by API server, it will return the API server's issued
 // certificate (pem-encoded). If there is any errors, or the watch timeouts, it
 // will return an error.
-func requestCertificate(client certificatesclient.CertificateSigningRequestInterface, csrData []byte, name string, usages []certificates.KeyUsage, privateKey interface{}) (certData []byte, err error) {
+func RequestCertificate(client certificatesclient.CertificateSigningRequestInterface, csrData []byte, name string, usages []certificates.KeyUsage, privateKey interface{}) (req *certificates.CertificateSigningRequest, err error) {
 	csr := &certificates.CertificateSigningRequest{
 		// Username, UID, Groups will be injected by API server.
 		TypeMeta: metav1.TypeMeta{Kind: "CertificateSigningRequest"},
@@ -89,28 +94,35 @@ func requestCertificate(client certificatesclient.CertificateSigningRequestInter
 			Usages:  usages,
 		},
 	}
+	if len(csr.Name) == 0 {
+		csr.GenerateName = "csr-"
+	}
 
-	req, err := client.Create(csr)
+	req, err = client.Create(csr)
 	switch {
 	case err == nil:
-	case errors.IsAlreadyExists(err):
+	case errors.IsAlreadyExists(err) && len(name) > 0:
 		glog.Infof("csr for this node already exists, reusing")
 		req, err = client.Get(name, metav1.GetOptions{})
 		if err != nil {
-			return nil, fmt.Errorf("cannot retrieve certificate signing request: %v", err)
+			return nil, formatError("cannot retrieve certificate signing request: %v", err)
 		}
 		if err := ensureCompatible(req, csr, privateKey); err != nil {
 			return nil, fmt.Errorf("retrieved csr is not compatible: %v", err)
 		}
 		glog.Infof("csr for this node is still valid")
 	default:
-		return nil, fmt.Errorf("cannot create certificate signing request: %v", err)
+		return nil, formatError("cannot create certificate signing request: %v", err)
 	}
+	return req, nil
+}
 
+// WaitForCertificate waits for a certificate to be issued until timeout, or returns an error.
+func WaitForCertificate(client certificatesclient.CertificateSigningRequestInterface, req *certificates.CertificateSigningRequest, timeout time.Duration) (certData []byte, err error) {
 	fieldSelector := fields.OneTermEqualSelector("metadata.name", req.Name).String()
 
 	event, err := cache.ListWatchUntil(
-		3600*time.Second,
+		timeout,
 		&cache.ListWatch{
 			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
 				options.FieldSelector = fieldSelector
@@ -124,6 +136,8 @@ func requestCertificate(client certificatesclient.CertificateSigningRequestInter
 		func(event watch.Event) (bool, error) {
 			switch event.Type {
 			case watch.Modified, watch.Added:
+			case watch.Deleted:
+				return false, fmt.Errorf("csr %q was deleted", req.Name)
 			default:
 				return false, nil
 			}
@@ -142,12 +156,14 @@ func requestCertificate(client certificatesclient.CertificateSigningRequestInter
 			return false, nil
 		},
 	)
+	if err == wait.ErrWaitTimeout {
+		return nil, wait.ErrWaitTimeout
+	}
 	if err != nil {
-		return nil, fmt.Errorf("cannot watch on the certificate signing request: %v", err)
+		return nil, formatError("cannot watch on the certificate signing request: %v", err)
 	}
 
 	return event.Object.(*certificates.CertificateSigningRequest).Status.Certificate, nil
-
 }
 
 // This digest should include all the relevant pieces of the CSR we care about.
@@ -184,11 +200,11 @@ func digestedName(privateKeyData []byte, subject *pkix.Name, usages []certificat
 
 // ensureCompatible ensures that a CSR object is compatible with an original CSR
 func ensureCompatible(new, orig *certificates.CertificateSigningRequest, privateKey interface{}) error {
-	newCsr, err := certhelper.ParseCSR(new)
+	newCsr, err := ParseCSR(new)
 	if err != nil {
 		return fmt.Errorf("unable to parse new csr: %v", err)
 	}
-	origCsr, err := certhelper.ParseCSR(orig)
+	origCsr, err := ParseCSR(orig)
 	if err != nil {
 		return fmt.Errorf("unable to parse original csr: %v", err)
 	}
@@ -203,5 +219,43 @@ func ensureCompatible(new, orig *certificates.CertificateSigningRequest, private
 	if err := newCsr.CheckSignature(); err != nil {
 		return fmt.Errorf("error validating signature new CSR against old key: %v", err)
 	}
+	if len(new.Status.Certificate) > 0 {
+		certs, err := certutil.ParseCertsPEM(new.Status.Certificate)
+		if err != nil {
+			return fmt.Errorf("error parsing signed certificate for CSR: %v", err)
+		}
+		now := time.Now()
+		for _, cert := range certs {
+			if now.After(cert.NotAfter) {
+				return fmt.Errorf("one of the certificates for the CSR has expired: %s", cert.NotAfter)
+			}
+		}
+	}
 	return nil
 }
+
+// formatError preserves the type of an API message but alters the message. Expects
+// a single argument format string, and returns the wrapped error.
+func formatError(format string, err error) error {
+	if s, ok := err.(errors.APIStatus); ok {
+		se := &errors.StatusError{ErrStatus: s.Status()}
+		se.ErrStatus.Message = fmt.Sprintf(format, se.ErrStatus.Message)
+		return se
+	}
+	return fmt.Errorf(format, err)
+}
+
+// ParseCSR extracts the CSR from the API object and decodes it.
+func ParseCSR(obj *certificates.CertificateSigningRequest) (*x509.CertificateRequest, error) {
+	// extract PEM from request object
+	pemBytes := obj.Spec.Request
+	block, _ := pem.Decode(pemBytes)
+	if block == nil || block.Type != "CERTIFICATE REQUEST" {
+		return nil, fmt.Errorf("PEM block type must be CERTIFICATE REQUEST")
+	}
+	csr, err := x509.ParseCertificateRequest(block.Bytes)
+	if err != nil {
+		return nil, err
+	}
+	return csr, nil
+}