apiserver: use fixtures for self-signed certs in test server

cmd/kube-apiserver/app/testing/BUILD

@@ -8,6 +8,7 @@ load(
 go_library(
     name = "go_default_library",
     srcs = ["testserver.go"],
+    data = glob(["testdata/**"]),
     importpath = "k8s.io/kubernetes/cmd/kube-apiserver/app/testing",
     deps = [
         "//cmd/kube-apiserver/app:go_default_library",

cmd/kube-apiserver/app/testing/testdata/127.0.0.1_10.0.0.1_kubernetes.default.svc-kubernetes.default-kubernetes-localhost.crt

+-----BEGIN CERTIFICATE-----
+MIIDTzCCAjegAwIBAgIBAjANBgkqhkiG9w0BAQsFADAiMSAwHgYDVQQDDBcxMjcu
+MC4wLjEtY2FAMTUzMTQ2NzU5MzAgFw0xODA3MTMwNjM5NTNaGA8yMTE4MDYxOTA2
+Mzk1M1owHzEdMBsGA1UEAwwUMTI3LjAuMC4xQDE1MzE0Njc1OTMwggEiMA0GCSqG
+SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDdTNF7rRKBvDtLOvCTDgj3utt+zv5u+23x
+kCz2475DPnTZ7JK2ipCuHemyCY88M6VyaBkIqAVvvl3LZiS+Hu3gd+8elbdGrCxQ
+sui1MrUcAg8OoBM+97UzoKC3HMFIFEpqzKjVJKr5PbV3F8XXIBQeS3YUCePo3m7u
+OkGCXUXtWRtQTu4Dcq+tJKlJBsY+Q8CUvb1l0n5hafIFEMnFF/sKGP28CWd8gfzD
+ZKKtVumvQlgcp1GdfxqKHfjQOtBo+ZBFiHgDGDrrghuQ2CxROvk5/bNrViqbWbw4
+lUbU3Yn18L4UHR5xOOvQyLP2QdWAaoPutT7Xba40RMgWYlsNRaatAgMBAAGjgZAw
+gY0wDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB
+/wQCMAAwWAYDVR0RBFEwT4IWa3ViZXJuZXRlcy5kZWZhdWx0LnN2Y4ISa3ViZXJu
+ZXRlcy5kZWZhdWx0ggprdWJlcm5ldGVzgglsb2NhbGhvc3SHBH8AAAGHBAoAAAEw
+DQYJKoZIhvcNAQELBQADggEBAFkRV1oBLrY3IJDI6E9nxAK30EdyfYZqvybPCZB8
+6AAErj+WleJVFi0rZJ3fRDoQ5Gelwe4Ud21DknW4+L7nZ8JRbzNkLTYTJxtkujSW
+aEz7xKW1IxD+o9TEceqiVko4xGawXjUVTun7n0Upv6T4D4jC0GN9zu8oT6xbUHmd
+WSSc2HjGLs8vF130xt2Oj0jx03i7AoJF4ZxMRt7dqSK7j5tfflfTS9Dxhmd9Gg5P
+eGH4BWJ3IJI3r0+WUtiIgMSgV2ppTSNY2UNbNNpudsRCq55IzyHuRioFt/FH9t+8
+xFaar6D9RDsm87JCv5JZ3BoVZJglmX8iqye+OBXgHgMZxx4=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIC5DCCAcygAwIBAgIBATANBgkqhkiG9w0BAQsFADAiMSAwHgYDVQQDDBcxMjcu
+MC4wLjEtY2FAMTUzMTQ2NzU5MzAgFw0xODA3MTMwNjM5NTNaGA8yMTE4MDYxOTA2
+Mzk1M1owIjEgMB4GA1UEAwwXMTI3LjAuMC4xLWNhQDE1MzE0Njc1OTMwggEiMA0G
+CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDRfS+rc4EPpDafeiRo+uIvV1812UQM
+ddoaCrn2yIVSYiupsFc7goieXOpqxgI6ksUCMDUOfi3DQGC8067wX2HpMzz5J9yz
+Qfamcg3SL7G9u5Vx+x+EU6qmBhXa4Z46JwTY0vYeccz2PR+Nx+HHO0DglIh3tip8
+ECQ2rtpMc5YxJOCwJg3zh8pnEqLNEahm3p1lNGLbY7Kpqp7al68ZVReVg/YaoJt5
+Voi7vbR38OWBChbBmwKRP4gJD8aKY2eY6Xgn8+UAAytYGOEp18y/eAvba7awKp56
+wG1Y3JqWD06D8NnUCPQOO/g/KyGU77sM66xdlsOwLpSbtwWLcjC3nnvHAgMBAAGj
+IzAhMA4GA1UdDwEB/wQEAwICpDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEB
+CwUAA4IBAQCECJvqzrh57MrVT5sj+TDmhtSwkblBBqNG03X8gvmj7PsYAnZVpaev
+KbN0xfDhK6yaCJX41sZCoTaQa55a0Y9n/Lr6d2LREYPf2NdhHU2aj1UjNhhamoZk
+0/MJtn/7t6UmYsdFIRlYtLJZQRfNaeO+ULpjjQeGj+Y4mR87VzyDZk2zi/fLJCtk
+aVKsI2Tan9KFzwmsCp/9RH7uPhOIFsaa8ePBCvzrahMrG+b9FGV670bQTS104Gyt
+HB73ixOheUPL9PuvahXKz0xlJfeKu4nFFJkqUmThj2Ybv8cyzDNrSyDywZxzNe3e
+nMA3i/kfmIj33gkmwcFgYPqfKleeVZQo
+-----END CERTIFICATE-----

cmd/kube-apiserver/app/testing/testdata/127.0.0.1_10.0.0.1_kubernetes.default.svc-kubernetes.default-kubernetes-localhost.key

+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA3UzRe60Sgbw7Szrwkw4I97rbfs7+bvtt8ZAs9uO+Qz502eyS
+toqQrh3psgmPPDOlcmgZCKgFb75dy2Ykvh7t4HfvHpW3RqwsULLotTK1HAIPDqAT
+Pve1M6CgtxzBSBRKasyo1SSq+T21dxfF1yAUHkt2FAnj6N5u7jpBgl1F7VkbUE7u
+A3KvrSSpSQbGPkPAlL29ZdJ+YWnyBRDJxRf7Chj9vAlnfIH8w2SirVbpr0JYHKdR
+nX8aih340DrQaPmQRYh4Axg664IbkNgsUTr5Of2za1Yqm1m8OJVG1N2J9fC+FB0e
+cTjr0Miz9kHVgGqD7rU+122uNETIFmJbDUWmrQIDAQABAoIBAFXzdhFhASUeZLEt
+bS7Qbq85BfNUlgGo6eS+qJgjkEwxv9S8S6dWXiciOxgJGna5YHL093QjPilOeMN9
+IpwtCxr5ugfZAlwSlwuo0TU/QpRkQFDf31m/f8NTidhU9MT4DIc6ggB2w2kWjJp6
+wz5wmR/DE1NpG/ngGpmwSq1FaNlr3xz4e6b0A56ReqQr5YwYsZl2Fxf8sOBWTiPe
+Iv41q8jyRXL2ytv9uTgdD7i+qLMz1/NGvy4ZWxD3yCMsDm5eEI8/4l2pOmRrrpKY
+Fc14eUkbHBMyT6ibI4d7Y2aZJslq8d0HMMX1XNLvzLEnGT1+mrOjWwerI+60B0t1
+6EvTfUkCgYEA/rVROq6JupfnT7BM04jEx5UuaokcLoNpn6bptZwmMakioLjgZoa2
+XEZrNoRWVMQ82JuguxkLtUgLgqKQRreolDCyQtaGFjFnWdVs+1W0oIHY7oMdwOLh
+XsQRtPW3HCK4HYZJcBBIh8t4USiBTrRtTpXmDinLkbCRXYOcehbRZ2cCgYEA3mwg
+tsbnSDYTcFmPjJEGjISfMEjAHoka8ubPKzU6VylDnrb2k/NbDYL3NidzGbJaVJFk
+YNfCsja4COz+0pBiMY2fBEzHU4RwDaRrxUr0fLVxvH7/E9JPP8y/e5CJR2Z2sDQa
+yed3ArkNh0MaecGr+7IZFbv+Uj4QaBq3W77hGMsCgYB/keC1O2XQBvTbfLl92Sp1
+q8orobBzu23TgI3dX+/hP40QYohB0YbUgJCCJZX3rrGq64d9LfYnPYZGT5VjVekh
+D6K4xykxRF03KSYEW9Cz81TrYNAuI3QtOpaDw+2KMfl1ECUH85/gI5CHVXouKT/1
+9C3dOiGzPnQQGjLtEzCeUQKBgFacZGDIM2e7Jvao6W0jTBmLyzFSIv3BBe1wU1vP
+7lfiiaJUPNCAAwTP6tP7qKZo/SPROfU8D2S2ShOvtcrozlPdgf56p2OuPrQRQqYg
++fNV9GQiT9G4I4QEhsvnDI3xKGaU45mbuIwm4024o6al9AKe54W/HtmHsXvYa24e
+dijhAoGARcbgcE/aT8jhdVHHCRBuSD4ZzXbB+JCetHsrjhOYnifc0graq0umiuRI
+c0i+IT5OhGTdVbjnPgySHn/V/IuSYLLtKvfqSV8tQk3womXRPJ/K9BsFhelo1Vd5
+MTyZ2j0XjLWHOo0DKxIPLW3P7sBYAFM2Z+/RAe1uKjISmggDhBs=
+-----END RSA PRIVATE KEY-----

cmd/kube-apiserver/app/testing/testserver.go

@@ -21,6 +21,8 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"path"
+	"runtime"
 	"time"
 
 	pflag "github.com/spf13/pflag"
@@ -111,11 +113,18 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo
 
 	s.InsecureServing.BindPort = 0
 
-	s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort()
+	s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
 	if err != nil {
 		return result, fmt.Errorf("failed to create listener: %v", err)
 	}
 	s.SecureServing.ServerCert.CertDirectory = result.TmpDir
+
+	_, thisFile, _, ok := runtime.Caller(0)
+	if !ok {
+		return result, fmt.Errorf("failed to get current file")
+	}
+	s.SecureServing.ServerCert.FixtureDirectory = path.Join(path.Dir(thisFile), "testdata")
+
 	s.ServiceClusterIPRange.IP = net.IPv4(10, 0, 0, 0)
 	s.ServiceClusterIPRange.Mask = net.CIDRMask(16, 32)
 	s.Etcd.StorageConfig = *storageConfig
@@ -132,7 +141,6 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo
 	server, err := app.CreateServerChain(completedOptions, stopCh)
 	if err != nil {
 		return result, fmt.Errorf("failed to create server chain: %v", err)
-
 	}
 	go func(stopCh <-chan struct{}) {
 		if err := server.PrepareRun().Run(stopCh); err != nil {
@@ -178,8 +186,8 @@ func StartTestServerOrDie(t Logger, instanceOptions *TestServerInstanceOptions,
 	return nil
 }
 
-func createListenerOnFreePort() (net.Listener, int, error) {
-	ln, err := net.Listen("tcp", ":0")
+func createLocalhostListenerOnFreePort() (net.Listener, int, error) {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		return nil, 0, err
 	}

staging/src/k8s.io/apiserver/pkg/server/options/serving.go

@@ -77,6 +77,11 @@ type GeneratableKeyCert struct {
 	// CertDirectory is a directory that will contain the certificates.  If the cert and key aren't specifically set
 	// this will be used to derive a match with the "pair-name"
 	CertDirectory string
+	// FixtureDirectory is a directory that contains test fixture used to avoid regeneration of certs during tests.
+	// The format is:
+	// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.crt
+	// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.key
+	FixtureDirectory string
 	// PairName is the name which will be used with CertDirectory to make a cert and key names
 	// It becomes CertDirector/PairName.crt and CertDirector/PairName.key
 	PairName string
@@ -269,7 +274,7 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str
 			alternateIPs = append(alternateIPs, s.BindAddress)
 		}
 
-		if cert, key, err := certutil.GenerateSelfSignedCertKey(publicAddress, alternateIPs, alternateDNS); err != nil {
+		if cert, key, err := certutil.GenerateSelfSignedCertKeyWithFixtures(publicAddress, alternateIPs, alternateDNS, s.ServerCert.FixtureDirectory); err != nil {
 			return fmt.Errorf("unable to generate self signed cert: %v", err)
 		} else {
 			if err := certutil.WriteCert(keyCert.CertFile, cert); err != nil {

staging/src/k8s.io/client-go/util/cert/cert.go

@@ -27,9 +27,12 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"math"
 	"math/big"
 	"net"
+	"path"
+	"strings"
 	"time"
 )
 
@@ -136,8 +139,38 @@ func MakeEllipticPrivateKeyPEM() ([]byte, error) {
 
 // GenerateSelfSignedCertKey creates a self-signed certificate and key for the given host.
 // Host may be an IP or a DNS name
-// You may also specify additional subject alt names (either ip or dns names) for the certificate
+// You may also specify additional subject alt names (either ip or dns names) for the certificate.
 func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS []string) ([]byte, []byte, error) {
+	return GenerateSelfSignedCertKeyWithFixtures(host, alternateIPs, alternateDNS, "")
+}
+
+// GenerateSelfSignedCertKeyWithFixtures creates a self-signed certificate and key for the given host.
+// Host may be an IP or a DNS name. You may also specify additional subject alt names (either ip or dns names)
+// for the certificate.
+//
+// If fixtureDirectory is non-empty, it is a directory path which can contain pre-generated certs. The format is:
+// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.crt
+// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.key
+// Certs/keys not existing in that directory are created.
+func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, alternateDNS []string, fixtureDirectory string) ([]byte, []byte, error) {
+	validFrom := time.Now().Add(-time.Hour) // valid an hour earlier to avoid flakes due to clock skew
+	maxAge := time.Hour * 24 * 365          // one year self-signed certs
+
+	baseName := fmt.Sprintf("%s_%s_%s", host, strings.Join(ipsToStrings(alternateIPs), "-"), strings.Join(alternateDNS, "-"))
+	certFixturePath := path.Join(fixtureDirectory, baseName+".crt")
+	keyFixturePath := path.Join(fixtureDirectory, baseName+".key")
+	if len(fixtureDirectory) > 0 {
+		cert, err := ioutil.ReadFile(certFixturePath)
+		if err == nil {
+			key, err := ioutil.ReadFile(keyFixturePath)
+			if err == nil {
+				return cert, key, nil
+			}
+			return nil, nil, fmt.Errorf("cert %s can be read, but key %s cannot: %v", certFixturePath, keyFixturePath, err)
+		}
+		maxAge = 100 * time.Hour * 24 * 365 // 100 years fixtures
+	}
+
 	caKey, err := rsa.GenerateKey(cryptorand.Reader, 2048)
 	if err != nil {
 		return nil, nil, err
@@ -148,8 +181,8 @@ func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS
 		Subject: pkix.Name{
 			CommonName: fmt.Sprintf("%s-ca@%d", host, time.Now().Unix()),
 		},
-		NotBefore: time.Now(),
-		NotAfter:  time.Now().Add(time.Hour * 24 * 365),
+		NotBefore: validFrom,
+		NotAfter:  validFrom.Add(maxAge),
 
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
 		BasicConstraintsValid: true,
@@ -176,8 +209,8 @@ func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS
 		Subject: pkix.Name{
 			CommonName: fmt.Sprintf("%s@%d", host, time.Now().Unix()),
 		},
-		NotBefore: time.Now(),
-		NotAfter:  time.Now().Add(time.Hour * 24 * 365),
+		NotBefore: validFrom,
+		NotAfter:  validFrom.Add(maxAge),
 
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
@@ -213,6 +246,15 @@ func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS
 		return nil, nil, err
 	}
 
+	if len(fixtureDirectory) > 0 {
+		if err := ioutil.WriteFile(certFixturePath, certBuffer.Bytes(), 0644); err != nil {
+			return nil, nil, fmt.Errorf("failed to write cert fixture to %s: %v", certFixturePath, err)
+		}
+		if err := ioutil.WriteFile(keyFixturePath, keyBuffer.Bytes(), 0644); err != nil {
+			return nil, nil, fmt.Errorf("failed to write key fixture to %s: %v", certFixturePath, err)
+		}
+	}
+
 	return certBuffer.Bytes(), keyBuffer.Bytes(), nil
 }
 
@@ -243,3 +285,11 @@ func FormatCert(c *x509.Certificate) string {
 	}
 	return res
 }
+
+func ipsToStrings(ips []net.IP) []string {
+	ss := make([]string, 0, len(ips))
+	for _, ip := range ips {
+		ss = append(ss, ip.String())
+	}
+	return ss
+}