[GCE Ingress e2e] Add test for pre-shared certificate

test/e2e/framework/ingress_utils.go

@@ -26,7 +26,6 @@ import (
 	"encoding/json"
 	"encoding/pem"
 	"fmt"
-	"io"
 	"math/big"
 	"net"
 	"net/http"
@@ -64,11 +63,20 @@ const (
 	// Ingress class annotation defined in ingress repository.
 	// TODO: All these annotations should be reused from
 	// ingress-gce/pkg/annotations instead of duplicating them here.
-	IngressClass = "kubernetes.io/ingress.class"
+	IngressClassKey = "kubernetes.io/ingress.class"
 
 	// Ingress class annotation value for multi cluster ingress.
 	MulticlusterIngressClassValue = "gce-multi-cluster"
 
+	// Static IP annotation defined in ingress repository.
+	IngressStaticIPKey = "kubernetes.io/ingress.global-static-ip-name"
+
+	// Allow HTTP annotation defined in ingress repository.
+	IngressAllowHTTPKey = "kubernetes.io/ingress.allow-http"
+
+	// Pre-shared-cert annotation defined in ingress repository.
+	IngressPreSharedCertKey = "ingress.gcp.kubernetes.io/pre-shared-cert"
+
 	// all cloud resources created by the ingress controller start with this
 	// prefix.
 	k8sPrefix = "k8s-"
@@ -210,15 +218,15 @@ func CreateIngressComformanceTests(jig *IngressTestJig, ns string, annotations m
 	}
 }
 
-// generateRSACerts generates a basic self signed certificate using a key length
+// GenerateRSACerts generates a basic self signed certificate using a key length
 // of rsaBits, valid for validFor time.
-func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error {
+func GenerateRSACerts(host string, isCA bool) ([]byte, []byte, error) {
 	if len(host) == 0 {
-		return fmt.Errorf("Require a non-empty host for client hello")
+		return nil, nil, fmt.Errorf("Require a non-empty host for client hello")
 	}
 	priv, err := rsa.GenerateKey(rand.Reader, rsaBits)
 	if err != nil {
-		return fmt.Errorf("Failed to generate key: %v", err)
+		return nil, nil, fmt.Errorf("Failed to generate key: %v", err)
 	}
 	notBefore := time.Now()
 	notAfter := notBefore.Add(validFor)
@@ -227,7 +235,7 @@ func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error {
 	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
 
 	if err != nil {
-		return fmt.Errorf("failed to generate serial number: %s", err)
+		return nil, nil, fmt.Errorf("failed to generate serial number: %s", err)
 	}
 	template := x509.Certificate{
 		SerialNumber: serialNumber,
@@ -257,17 +265,18 @@ func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error {
 		template.KeyUsage |= x509.KeyUsageCertSign
 	}
 
+	var keyOut, certOut bytes.Buffer
 	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
 	if err != nil {
-		return fmt.Errorf("Failed to create certificate: %s", err)
+		return nil, nil, fmt.Errorf("Failed to create certificate: %s", err)
 	}
-	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
-		return fmt.Errorf("Failed creating cert: %v", err)
+	if err := pem.Encode(&certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		return nil, nil, fmt.Errorf("Failed creating cert: %v", err)
 	}
-	if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
-		return fmt.Errorf("Failed creating keay: %v", err)
+	if err := pem.Encode(&keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
+		return nil, nil, fmt.Errorf("Failed creating keay: %v", err)
 	}
-	return nil
+	return certOut.Bytes(), keyOut.Bytes(), nil
 }
 
 // buildTransportWithCA creates a transport for use in executing HTTPS requests with
@@ -297,16 +306,13 @@ func BuildInsecureClient(timeout time.Duration) *http.Client {
 // If a secret with the same name already pathExists in the namespace of the
 // Ingress, it's updated.
 func createIngressTLSSecret(kubeClient clientset.Interface, ing *extensions.Ingress) (host string, rootCA, privKey []byte, err error) {
-	var k, c bytes.Buffer
 	tls := ing.Spec.TLS[0]
 	host = strings.Join(tls.Hosts, ",")
 	Logf("Generating RSA cert for host %v", host)
-
-	if err = generateRSACerts(host, true, &k, &c); err != nil {
+	cert, key, err := GenerateRSACerts(host, true)
+	if err != nil {
 		return
 	}
-	cert := c.Bytes()
-	key := k.Bytes()
 	secret := &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: tls.SecretName,
@@ -1046,7 +1052,7 @@ func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations m
 	j.Ingress, err = manifest.IngressFromManifest(filepath.Join(manifestPath, "ing.yaml"))
 	ExpectNoError(err)
 	j.Ingress.Namespace = ns
-	j.Ingress.Annotations = map[string]string{IngressClass: j.Class}
+	j.Ingress.Annotations = map[string]string{IngressClassKey: j.Class}
 	for k, v := range ingAnnotations {
 		j.Ingress.Annotations[k] = v
 	}
@@ -1145,6 +1151,31 @@ func WaitForIngressAddress(c clientset.Interface, ns, ingName string, timeout ti
 	return address, err
 }
 
+func (j *IngressTestJig) PollIngressWithCert(waitForNodePort bool, knownHosts []string, cert []byte) {
+	// Check that all rules respond to a simple GET.
+	knownHostsSet := sets.NewString(knownHosts...)
+	for _, rules := range j.Ingress.Spec.Rules {
+		timeoutClient := &http.Client{Timeout: IngressReqTimeout}
+		proto := "http"
+		if knownHostsSet.Has(rules.Host) {
+			var err error
+			// Create transport with cert to verify if the server uses the correct one.
+			timeoutClient.Transport, err = buildTransportWithCA(rules.Host, cert)
+			ExpectNoError(err)
+			proto = "https"
+		}
+		for _, p := range rules.IngressRuleValue.HTTP.Paths {
+			if waitForNodePort {
+				j.pollServiceNodePort(j.Ingress.Namespace, p.Backend.ServiceName, int(p.Backend.ServicePort.IntVal))
+			}
+			route := fmt.Sprintf("%v://%v%v", proto, j.Address, p.Path)
+			Logf("Testing route %v host %v with simple GET", route, rules.Host)
+			ExpectNoError(PollURL(route, rules.Host, LoadBalancerPollTimeout, j.PollInterval, timeoutClient, false))
+		}
+	}
+	Logf("Finished polling on all rules on ingress %q", j.Ingress.Name)
+}
+
 // WaitForIngress waits till the ingress acquires an IP, then waits for its
 // hosts/urls to respond to a protocol check (either http or https). If
 // waitForNodePort is true, the NodePort of the Service is verified before
@@ -1158,28 +1189,31 @@ func (j *IngressTestJig) WaitForIngress(waitForNodePort bool) {
 	}
 	j.Address = address
 	Logf("Found address %v for ingress %v", j.Address, j.Ingress.Name)
-	timeoutClient := &http.Client{Timeout: IngressReqTimeout}
 
-	// Check that all rules respond to a simple GET.
-	for _, rules := range j.Ingress.Spec.Rules {
-		proto := "http"
-		if len(j.Ingress.Spec.TLS) > 0 {
-			knownHosts := sets.NewString(j.Ingress.Spec.TLS[0].Hosts...)
-			if knownHosts.Has(rules.Host) {
-				timeoutClient.Transport, err = buildTransportWithCA(rules.Host, j.GetRootCA(j.Ingress.Spec.TLS[0].SecretName))
-				ExpectNoError(err)
-				proto = "https"
-			}
-		}
-		for _, p := range rules.IngressRuleValue.HTTP.Paths {
-			if waitForNodePort {
-				j.pollServiceNodePort(j.Ingress.Namespace, p.Backend.ServiceName, int(p.Backend.ServicePort.IntVal))
-			}
-			route := fmt.Sprintf("%v://%v%v", proto, address, p.Path)
-			Logf("Testing route %v host %v with simple GET", route, rules.Host)
-			ExpectNoError(PollURL(route, rules.Host, LoadBalancerPollTimeout, j.PollInterval, timeoutClient, false))
-		}
+	var knownHosts []string
+	var cert []byte
+	if len(j.Ingress.Spec.TLS) > 0 {
+		knownHosts = j.Ingress.Spec.TLS[0].Hosts
+		cert = j.GetRootCA(j.Ingress.Spec.TLS[0].SecretName)
+	}
+	j.PollIngressWithCert(waitForNodePort, knownHosts, cert)
+}
+
+// WaitForIngress waits till the ingress acquires an IP, then waits for its
+// hosts/urls to respond to a protocol check (either http or https). If
+// waitForNodePort is true, the NodePort of the Service is verified before
+// verifying the Ingress. NodePort is currently a requirement for cloudprovider
+// Ingress. Hostnames and certificate need to be explicitly passed in.
+func (j *IngressTestJig) WaitForIngressWithCert(waitForNodePort bool, knownHosts []string, cert []byte) {
+	// Wait for the loadbalancer IP.
+	address, err := WaitForIngressAddress(j.Client, j.Ingress.Namespace, j.Ingress.Name, LoadBalancerPollTimeout)
+	if err != nil {
+		Failf("Ingress failed to acquire an IP address within %v", LoadBalancerPollTimeout)
 	}
+	j.Address = address
+	Logf("Found address %v for ingress %v", j.Address, j.Ingress.Name)
+
+	j.PollIngressWithCert(waitForNodePort, knownHosts, cert)
 }
 
 // VerifyURL polls for the given iterations, in intervals, and fails if the

test/e2e/network/ingress.go

@@ -22,8 +22,11 @@ import (
 	"strings"
 	"time"
 
+	compute "google.golang.org/api/compute/v1"
+
 	extensions "k8s.io/api/extensions/v1beta1"
 	rbacv1beta1 "k8s.io/api/rbac/v1beta1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -34,7 +37,6 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	compute "google.golang.org/api/compute/v1"
 )
 
 const (
@@ -123,8 +125,8 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", ns, ip))
 
 			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns, map[string]string{
-				"kubernetes.io/ingress.global-static-ip-name": ns,
-				"kubernetes.io/ingress.allow-http":            "false",
+				framework.IngressStaticIPKey:  ns,
+				framework.IngressAllowHTTPKey: "false",
 			}, map[string]string{})
 
 			By("waiting for Ingress to come up with ip: " + ip)
@@ -317,10 +319,52 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			Expect(hcAfterSync.HttpHealthCheck.RequestPath).To(Equal(hcToChange.HttpHealthCheck.RequestPath))
 		})
 
+		It("should create ingress with pre-shared certificate", func() {
+			preSharedCertName := "test-pre-shared-cert"
+			By(fmt.Sprintf("Creating ssl certificate %q on GCE", preSharedCertName))
+			testHostname := "test.ingress.com"
+			cert, key, err := framework.GenerateRSACerts(testHostname, true)
+			Expect(err).NotTo(HaveOccurred())
+			gceCloud, err := framework.GetGCECloud()
+			Expect(err).NotTo(HaveOccurred())
+			defer func() {
+				// We would not be able to delete the cert until ingress controller
+				// cleans up the target proxy that references it.
+				By("Deleting ingress before deleting ssl certificate")
+				jig.TryDeleteIngress()
+				By(fmt.Sprintf("Deleting ssl certificate %q on GCE", preSharedCertName))
+				err := wait.Poll(framework.LoadBalancerPollInterval, framework.LoadBalancerCleanupTimeout, func() (bool, error) {
+					if err := gceCloud.DeleteSslCertificate(preSharedCertName); err != nil && !errors.IsNotFound(err) {
+						framework.Logf("Failed to delete ssl certificate %q: %v. Retrying...", preSharedCertName, err)
+						return false, nil
+					}
+					return true, nil
+				})
+				Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to delete ssl certificate %q: %v", preSharedCertName, err))
+			}()
+			_, err = gceCloud.CreateSslCertificate(&compute.SslCertificate{
+				Name:        preSharedCertName,
+				Certificate: string(cert),
+				PrivateKey:  string(key),
+				Description: "pre-shared cert for ingress testing",
+			})
+			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to create ssl certificate %q: %v", preSharedCertName, err))
+
+			By("Creating an ingress referencing the pre-shared certificate")
+			// Create an ingress referencing this cert using pre-shared-cert annotation.
+			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "pre-shared-cert"), ns, map[string]string{
+				framework.IngressPreSharedCertKey: preSharedCertName,
+				framework.IngressAllowHTTPKey:     "false",
+			}, map[string]string{})
+
+			By("Test that ingress works with the pre-shared certificate")
+			jig.WaitForIngressWithCert(true, []string{testHostname}, cert)
+		})
+
 		It("multicluster ingress should get instance group annotation", func() {
 			name := "echomap"
 			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "http"), ns, map[string]string{
-				framework.IngressClass: framework.MulticlusterIngressClassValue,
+				framework.IngressClassKey: framework.MulticlusterIngressClassValue,
 			}, map[string]string{})
 
 			By(fmt.Sprintf("waiting for Ingress %s to come up", name))
@@ -343,6 +387,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 		// TODO: Implement a multizone e2e that verifies traffic reaches each
 		// zone based on pod labels.
 	})
+
 	Describe("GCE [Slow] [Feature:NEG]", func() {
 		var gceController *framework.GCEIngressController
 

test/e2e/testing-manifests/ingress/pre-shared-cert/ing.yaml

+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: pre-shared-cert
+  # Below annotation will be added upon test:
+  # annotations:
+  #  ingress.gcp.kubernetes.io/pre-shared-cert: "test-pre-shared-cert"
+spec:
+  rules:
+  - host: test.ingress.com
+    http:
+      paths:
+      - path: /test
+        backend:
+          serviceName: echoheaders-https
+          servicePort: 80

test/e2e/testing-manifests/ingress/pre-shared-cert/rc.yaml

+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: echoheaders-https
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: echoheaders-https
+    spec:
+      containers:
+      - name: echoheaders-https
+        image: gcr.io/google_containers/echoserver:1.6
+        ports:
+        - containerPort: 8080

test/e2e/testing-manifests/ingress/pre-shared-cert/svc.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  name: echoheaders-https
+  labels:
+    app: echoheaders-https
+spec:
+  type: NodePort
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: echoheaders-https

test/e2e/upgrades/ingress.go

@@ -94,8 +94,8 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
 	// Create a working basic Ingress
 	By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", t.ipName, t.ip))
 	jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip-2"), ns.Name, map[string]string{
-		"kubernetes.io/ingress.global-static-ip-name": t.ipName,
-		"kubernetes.io/ingress.allow-http":            "false",
+		framework.IngressStaticIPKey:  t.ipName,
+		framework.IngressAllowHTTPKey: "false",
 	}, map[string]string{})
 	t.jig.AddHTTPS("tls-secret", "ingress.test.com")