Merge pull request #36546 from MrHohn/e2e-firewall

test/e2e/BUILD

@@ -51,6 +51,7 @@ go_library(
         "federation-replicaset.go",
         "federation-util.go",
         "federation-util-14.go",
+        "firewall.go",
         "garbage_collector.go",
         "generated_clientset.go",
         "gke_local_ssd.go",

test/e2e/firewall.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api/v1"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
+	"k8s.io/kubernetes/pkg/master/ports"
+	"k8s.io/kubernetes/pkg/util/sets"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	firewallTimeoutDefault = 3 * time.Minute
+	firewallTestTcpTimeout = time.Duration(1 * time.Second)
+	// Set ports outside of 30000-32767, 80 and 8080 to avoid being whitelisted by the e2e cluster
+	firewallTestHttpPort = int32(29999)
+	firewallTestUdpPort  = int32(29998)
+)
+
+var _ = framework.KubeDescribe("Firewall rule", func() {
+	var firewall_test_name = "firewall-test"
+	f := framework.NewDefaultFramework(firewall_test_name)
+
+	var cs clientset.Interface
+	var cloudConfig framework.CloudConfig
+	var gceCloud *gcecloud.GCECloud
+
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("gce")
+		cs = f.ClientSet
+		cloudConfig = framework.TestContext.CloudConfig
+		gceCloud = cloudConfig.Provider.(*gcecloud.GCECloud)
+	})
+
+	// This test takes around 4 minutes to run
+	It("[Slow] [Serial] should create valid firewall rules for LoadBalancer type service", func() {
+		ns := f.Namespace.Name
+		// This source ranges is just used to examine we have exact same things on LB firewall rules
+		firewallTestSourceRanges := []string{"0.0.0.0/1", "128.0.0.0/1"}
+		serviceName := "firewall-test-loadbalancer"
+
+		jig := NewServiceTestJig(cs, serviceName)
+		nodesNames := jig.GetNodesNames(maxNodesForEndpointsTests)
+		if len(nodesNames) <= 0 {
+			framework.Failf("Expect at least 1 node, got: %v", nodesNames)
+		}
+		nodesSet := sets.NewString(nodesNames...)
+
+		// OnlyLocal service is needed to examine which exact nodes the requests are being forwarded to by the Load Balancer on GCE
+		By("Creating a LoadBalancer type service with onlyLocal annotation")
+		svc := jig.createOnlyLocalLoadBalancerService(ns, serviceName,
+			loadBalancerCreateTimeoutDefault, false, func(svc *v1.Service) {
+				svc.Spec.Ports = []v1.ServicePort{{Protocol: "TCP", Port: firewallTestHttpPort}}
+				svc.Spec.LoadBalancerSourceRanges = firewallTestSourceRanges
+			})
+		defer func() {
+			jig.UpdateServiceOrFail(svc.Namespace, svc.Name, func(svc *v1.Service) {
+				svc.Spec.Type = v1.ServiceTypeNodePort
+				svc.Spec.LoadBalancerSourceRanges = nil
+			})
+			Expect(cs.Core().Services(svc.Namespace).Delete(svc.Name, nil)).NotTo(HaveOccurred())
+		}()
+		svcExternalIP := svc.Status.LoadBalancer.Ingress[0].IP
+
+		By("Checking if service's firewall rules are correct")
+		nodeTags := framework.GetInstanceTags(cloudConfig, nodesNames[0])
+		expFw := framework.ConstructFirewallForLBService(svc, nodeTags.Items)
+		fw, err := gceCloud.GetFirewall(expFw.Name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(framework.VerifyFirewallRule(fw, expFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
+
+		By(fmt.Sprintf("Creating netexec pods on at most %v nodes", maxNodesForEndpointsTests))
+		for i, nodeName := range nodesNames {
+			podName := fmt.Sprintf("netexec%v", i)
+			jig.LaunchNetexecPodOnNode(f, nodeName, podName, firewallTestHttpPort, firewallTestUdpPort, true)
+			defer func() {
+				framework.Logf("Cleaning up the netexec pod: %v", podName)
+				Expect(cs.Core().Pods(ns).Delete(podName, nil)).NotTo(HaveOccurred())
+			}()
+		}
+
+		// Send requests from outside of the cluster because internal traffic is whitelisted
+		By("Accessing the external service ip from outside, all non-master nodes should be reached")
+		Expect(testHitNodesFromOutside(svcExternalIP, firewallTestHttpPort, firewallTimeoutDefault, nodesSet)).NotTo(HaveOccurred())
+
+		// Check if there are overlapping tags on the firewall that extend beyond just the vms in our cluster
+		// by removing the tag on one vm and make sure it doesn't get any traffic. This is an imperfect
+		// simulation, we really want to check that traffic doesn't reach a vm outside the GKE cluster, but
+		// that's much harder to do in the current e2e framework.
+		By("Removing tags from one of the nodes")
+		nodesSet.Delete(nodesNames[0])
+		removedTags := framework.SetInstanceTags(cloudConfig, nodesNames[0], []string{})
+		defer func() {
+			By("Adding tags back to the node and wait till the traffic is recovered")
+			nodesSet.Insert(nodesNames[0])
+			framework.SetInstanceTags(cloudConfig, nodesNames[0], removedTags)
+			// Make sure traffic is recovered before exit
+			Expect(testHitNodesFromOutside(svcExternalIP, firewallTestHttpPort, firewallTimeoutDefault, nodesSet)).NotTo(HaveOccurred())
+		}()
+
+		By("Accessing serivce through the external ip and examine got no response from the node without tags")
+		Expect(testHitNodesFromOutsideWithCount(svcExternalIP, firewallTestHttpPort, firewallTimeoutDefault, nodesSet, 15)).NotTo(HaveOccurred())
+	})
+
+	It("should have correct firewall rules for e2e cluster", func() {
+		By("Gathering firewall related information")
+		masterTags := framework.GetInstanceTags(cloudConfig, cloudConfig.MasterName)
+		Expect(len(masterTags.Items)).Should(Equal(1))
+
+		nodes := framework.GetReadySchedulableNodesOrDie(cs)
+		if len(nodes.Items) <= 0 {
+			framework.Failf("Expect at least 1 node, got: %v", len(nodes.Items))
+		}
+		nodeTags := framework.GetInstanceTags(cloudConfig, nodes.Items[0].Name)
+		Expect(len(nodeTags.Items)).Should(Equal(1))
+
+		By("Checking if e2e firewall rules are correct")
+		for _, expFw := range framework.GetE2eFirewalls(cloudConfig.MasterName, masterTags.Items[0], nodeTags.Items[0], cloudConfig.Network) {
+			fw, err := gceCloud.GetFirewall(expFw.Name)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(framework.VerifyFirewallRule(fw, expFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
+		}
+
+		By("Checking well known ports on master and nodes are not exposed externally")
+		nodeAddrs := framework.NodeAddresses(nodes, v1.NodeExternalIP)
+		Expect(len(nodeAddrs)).NotTo(BeZero())
+		masterAddr := framework.GetMasterAddress(cs)
+		flag, _ := testNotReachableHTTPTimeout(masterAddr, ports.ControllerManagerPort, firewallTestTcpTimeout)
+		Expect(flag).To(BeTrue())
+		flag, _ = testNotReachableHTTPTimeout(masterAddr, ports.SchedulerPort, firewallTestTcpTimeout)
+		Expect(flag).To(BeTrue())
+		flag, _ = testNotReachableHTTPTimeout(nodeAddrs[0], ports.KubeletPort, firewallTestTcpTimeout)
+		Expect(flag).To(BeTrue())
+		flag, _ = testNotReachableHTTPTimeout(nodeAddrs[0], ports.KubeletReadOnlyPort, firewallTestTcpTimeout)
+		Expect(flag).To(BeTrue())
+		flag, _ = testNotReachableHTTPTimeout(nodeAddrs[0], ports.ProxyStatusPort, firewallTestTcpTimeout)
+		Expect(flag).To(BeTrue())
+	})
+})

test/e2e/framework/BUILD

@@ -13,6 +13,7 @@ go_library(
         "cleanup.go",
         "exec_util.go",
         "federation_util.go",
+        "firewall_util.go",
         "framework.go",
         "get-kubemark-resource-usage.go",
         "google_compute.go",
@@ -100,6 +101,7 @@ go_library(
         "//vendor:github.com/spf13/viper",
         "//vendor:golang.org/x/crypto/ssh",
         "//vendor:golang.org/x/net/websocket",
+        "//vendor:google.golang.org/api/compute/v1",
         "//vendor:google.golang.org/api/googleapi",
         "//vendor:gopkg.in/yaml.v2",
         "//vendor:k8s.io/client-go/kubernetes",

test/e2e/ingress.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 )
 
 const (
@@ -119,7 +120,7 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() {
 			}
 		})
 
-		It("shoud create ingress with given static-ip ", func() {
+		It("shoud create ingress with given static-ip", func() {
 			// ip released when the rest of lb resources are deleted in cleanupGCE
 			ip := gceController.createStaticIP(ns)
 			By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", ns, ip))
@@ -136,6 +137,14 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() {
 			By("should reject HTTP traffic")
 			framework.ExpectNoError(pollURL(fmt.Sprintf("http://%v/", ip), "", lbPollTimeout, jig.pollInterval, httpClient, true))
 
+			By("should have correct firewall rule for ingress")
+			fw := gceController.getFirewallRule()
+			expFw := jig.constructFirewallForIngress(gceController)
+			// Passed the last argument as `true` to verify the backend ports is a subset
+			// of the allowed ports in firewall rule, given there may be other existing
+			// ingress resources and backends we are not aware of.
+			Expect(framework.VerifyFirewallRule(fw, expFw, gceController.cloud.Network, true)).NotTo(HaveOccurred())
+
 			// TODO: uncomment the restart test once we have a way to synchronize
 			// and know that the controller has resumed watching. If we delete
 			// the ingress before the controller is ready we will leak.

test/e2e/ingress_utils.go

@@ -34,6 +34,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -76,6 +77,12 @@ const (
 	// to split uid from other naming/metadata.
 	clusterDelimiter = "--"
 
+	// Name of the default http backend service
+	defaultBackendName = "default-http-backend"
+
+	// IP src range from which the GCE L7 performs health checks.
+	GCEL7SrcRange = "130.211.0.0/22"
+
 	// Cloud resources created by the ingress controller older than this
 	// are automatically purged to prevent running out of quota.
 	// TODO(37335): write soak tests and bump this up to a week.
@@ -604,6 +611,18 @@ func (cont *GCEIngressController) canDelete(resourceName, creationTimestamp stri
 	return false
 }
 
+func (cont *GCEIngressController) getFirewallRuleName() string {
+	return fmt.Sprintf("%vfw-l7%v%v", k8sPrefix, clusterDelimiter, cont.UID)
+}
+
+func (cont *GCEIngressController) getFirewallRule() *compute.Firewall {
+	gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud)
+	fwName := cont.getFirewallRuleName()
+	fw, err := gceCloud.GetFirewall(fwName)
+	Expect(err).NotTo(HaveOccurred())
+	return fw
+}
+
 func (cont *GCEIngressController) deleteFirewallRule(del bool) (msg string) {
 	fwList := []compute.Firewall{}
 	regex := fmt.Sprintf("%vfw-l7%v.*", k8sPrefix, clusterDelimiter)
@@ -897,6 +916,50 @@ func (j *testJig) curlServiceNodePort(ns, name string, port int) {
 	framework.ExpectNoError(pollURL(u, "", 30*time.Second, j.pollInterval, &http.Client{Timeout: reqTimeout}, false))
 }
 
+// getIngressNodePorts returns all related backend services' nodePorts.
+// Current GCE ingress controller allows traffic to the default HTTP backend
+// by default, so retrieve its nodePort as well.
+func (j *testJig) getIngressNodePorts() []string {
+	nodePorts := []string{}
+	defaultSvc, err := j.client.Core().Services(api.NamespaceSystem).Get(defaultBackendName, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	nodePorts = append(nodePorts, strconv.Itoa(int(defaultSvc.Spec.Ports[0].NodePort)))
+
+	backendSvcs := []string{}
+	if j.ing.Spec.Backend != nil {
+		backendSvcs = append(backendSvcs, j.ing.Spec.Backend.ServiceName)
+	}
+	for _, rule := range j.ing.Spec.Rules {
+		for _, ingPath := range rule.HTTP.Paths {
+			backendSvcs = append(backendSvcs, ingPath.Backend.ServiceName)
+		}
+	}
+	for _, svcName := range backendSvcs {
+		svc, err := j.client.Core().Services(j.ing.Namespace).Get(svcName, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+		nodePorts = append(nodePorts, strconv.Itoa(int(svc.Spec.Ports[0].NodePort)))
+	}
+	return nodePorts
+}
+
+// constructFirewallForIngress returns the expected GCE firewall rule for the ingress resource
+func (j *testJig) constructFirewallForIngress(gceController *GCEIngressController) *compute.Firewall {
+	nodeTags := framework.GetNodeTags(j.client, gceController.cloud)
+	nodePorts := j.getIngressNodePorts()
+
+	fw := compute.Firewall{}
+	fw.Name = gceController.getFirewallRuleName()
+	fw.SourceRanges = []string{GCEL7SrcRange}
+	fw.TargetTags = nodeTags.Items
+	fw.Allowed = []*compute.FirewallAllowed{
+		{
+			IPProtocol: "tcp",
+			Ports:      nodePorts,
+		},
+	}
+	return &fw
+}
+
 // ingFromManifest reads a .json/yaml file and returns the rc in it.
 func ingFromManifest(fileName string) *extensions.Ingress {
 	var ing extensions.Ingress