Merge pull request #64140 from dixudx/add_e2e_kubelet_port

test/e2e/auth/BUILD

@@ -12,12 +12,14 @@ go_library(
         "certificates.go",
         "framework.go",
         "metadata_concealment.go",
+        "node_authn.go",
         "node_authz.go",
         "pod_security_policy.go",
         "service_accounts.go",
     ],
     importpath = "k8s.io/kubernetes/test/e2e/auth",
     deps = [
+        "//pkg/master/ports:go_default_library",
         "//pkg/security/apparmor:go_default_library",
         "//pkg/security/podsecuritypolicy/seccomp:go_default_library",
         "//pkg/security/podsecuritypolicy/util:go_default_library",

test/e2e/auth/node_authn.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package auth
+
+import (
+	"fmt"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/master/ports"
+	"k8s.io/kubernetes/test/e2e/framework"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = SIGDescribe("[Feature:NodeAuthenticator]", func() {
+
+	f := framework.NewDefaultFramework("node-authn")
+	var ns string
+	var nodeIPs []string
+	BeforeEach(func() {
+		ns = f.Namespace.Name
+
+		nodeList, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(nodeList.Items)).NotTo(BeZero())
+
+		pickedNode := nodeList.Items[0]
+		nodeIPs = framework.GetNodeAddresses(&pickedNode, v1.NodeExternalIP)
+		// The pods running in the cluster can see the internal addresses.
+		nodeIPs = append(nodeIPs, framework.GetNodeAddresses(&pickedNode, v1.NodeInternalIP)...)
+
+		// make sure ServiceAccount admission controller is enabled, so secret generation on SA creation works
+		sa, err := f.ClientSet.CoreV1().ServiceAccounts(ns).Get("default", metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(sa.Secrets)).NotTo(BeZero())
+	})
+
+	It("The kubelet's main port 10250 should reject requests with no credentials", func() {
+		pod := createNodeAuthTestPod(f)
+		for _, nodeIP := range nodeIPs {
+			// Anonymous authentication is disabled by default
+			result := framework.RunHostCmdOrDie(ns, pod.Name, fmt.Sprintf("curl -sIk -o /dev/null -w '%s' https://%s:%v/metrics", "%{http_code}", nodeIP, ports.KubeletPort))
+			Expect(result).To(Or(Equal("401"), Equal("403")), "the kubelet's main port 10250 should reject requests with no credentials")
+		}
+	})
+
+	It("The kubelet can delegate ServiceAccount tokens to the API server", func() {
+		By("create a new ServiceAccount for authentication")
+		trueValue := true
+		newSA := &v1.ServiceAccount{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: ns,
+				Name:      "node-auth-newSA",
+			},
+			AutomountServiceAccountToken: &trueValue,
+		}
+		_, err := f.ClientSet.CoreV1().ServiceAccounts(ns).Create(newSA)
+		Expect(err).NotTo(HaveOccurred())
+
+		pod := createNodeAuthTestPod(f)
+
+		for _, nodeIP := range nodeIPs {
+			result := framework.RunHostCmdOrDie(ns,
+				pod.Name,
+				fmt.Sprintf("curl -sIk -o /dev/null -w '%s' --header \"Authorization: Bearer `%s`\" https://%s:%v/metrics",
+					"%{http_code}",
+					"cat /var/run/secrets/kubernetes.io/serviceaccount/token",
+					nodeIP, ports.KubeletPort))
+			Expect(result).To(Or(Equal("401"), Equal("403")), "the kubelet can delegate ServiceAccount tokens to the API server")
+		}
+	})
+})
+
+func createNodeAuthTestPod(f *framework.Framework) *v1.Pod {
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "test-node-authn-",
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{{
+				Name:    "test-node-authn",
+				Image:   imageutils.GetE2EImage(imageutils.Hostexec),
+				Command: []string{"sleep 3600"},
+			}},
+			RestartPolicy: v1.RestartPolicyNever,
+		},
+	}
+
+	return f.PodClient().CreateSync(pod)
+}

test/e2e/framework/kubelet_stats.go

@@ -492,7 +492,7 @@ type usageDataPerContainer struct {
 }
 
 func GetKubeletHeapStats(c clientset.Interface, nodeName string) (string, error) {
-	client, err := NodeProxyRequest(c, nodeName, "debug/pprof/heap")
+	client, err := NodeProxyRequest(c, nodeName, "debug/pprof/heap", ports.KubeletPort)
 	if err != nil {
 		return "", err
 	}

test/e2e/framework/util.go

@@ -4492,7 +4492,7 @@ func isElementOf(podUID types.UID, pods *v1.PodList) bool {
 const proxyTimeout = 2 * time.Minute
 
 // NodeProxyRequest performs a get on a node proxy endpoint given the nodename and rest client.
-func NodeProxyRequest(c clientset.Interface, node, endpoint string) (restclient.Result, error) {
+func NodeProxyRequest(c clientset.Interface, node, endpoint string, port int) (restclient.Result, error) {
 	// proxy tends to hang in some cases when Node is not ready. Add an artificial timeout for this call.
 	// This will leak a goroutine if proxy hangs. #22165
 	var result restclient.Result
@@ -4501,7 +4501,7 @@ func NodeProxyRequest(c clientset.Interface, node, endpoint string) (restclient.
 		result = c.CoreV1().RESTClient().Get().
 			Resource("nodes").
 			SubResource("proxy").
-			Name(fmt.Sprintf("%v:%v", node, ports.KubeletPort)).
+			Name(fmt.Sprintf("%v:%v", node, port)).
 			Suffix(endpoint).
 			Do()
 
@@ -4529,7 +4529,7 @@ func GetKubeletRunningPods(c clientset.Interface, node string) (*v1.PodList, err
 
 func getKubeletPods(c clientset.Interface, node, resource string) (*v1.PodList, error) {
 	result := &v1.PodList{}
-	client, err := NodeProxyRequest(c, node, resource)
+	client, err := NodeProxyRequest(c, node, resource, ports.KubeletPort)
 	if err != nil {
 		return &v1.PodList{}, err
 	}

test/e2e/lifecycle/BUILD

@@ -12,6 +12,7 @@ go_library(
         "cluster_upgrade.go",
         "framework.go",
         "ha_master.go",
+        "kubelet_security.go",
         "reboot.go",
         "resize_nodes.go",
         "restart.go",
@@ -20,6 +21,7 @@ go_library(
     deps = [
         "//pkg/apis/core:go_default_library",
         "//pkg/kubelet/pod:go_default_library",
+        "//pkg/master/ports:go_default_library",
         "//pkg/util/version:go_default_library",
         "//test/e2e/chaosmonkey:go_default_library",
         "//test/e2e/common:go_default_library",

test/e2e/lifecycle/kubelet_security.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lifecycle
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/kubernetes/pkg/master/ports"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = SIGDescribe("Ports Security Check [Feature:KubeletSecurity]", func() {
+	f := framework.NewDefaultFramework("kubelet-security")
+
+	var node *v1.Node
+	var nodeName string
+
+	BeforeEach(func() {
+		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+		Expect(len(nodes.Items)).NotTo(BeZero())
+		node = &nodes.Items[0]
+		nodeName = node.Name
+	})
+
+	// make sure kubelet readonly (10255) and cadvisor (4194) ports are disabled via API server proxy
+	It(fmt.Sprintf("should not be able to proxy to the readonly kubelet port %v using proxy subresource", ports.KubeletReadOnlyPort), func() {
+		result, err := framework.NodeProxyRequest(f.ClientSet, nodeName, "pods/", ports.KubeletReadOnlyPort)
+		Expect(err).NotTo(HaveOccurred())
+
+		var statusCode int
+		result.StatusCode(&statusCode)
+		Expect(statusCode).NotTo(Equal(http.StatusOK))
+	})
+	It("should not be able to proxy to cadvisor port 4194 using proxy subresource", func() {
+		result, err := framework.NodeProxyRequest(f.ClientSet, nodeName, "containers/", 4194)
+		Expect(err).NotTo(HaveOccurred())
+
+		var statusCode int
+		result.StatusCode(&statusCode)
+		Expect(statusCode).NotTo(Equal(http.StatusOK))
+	})
+
+	// make sure kubelet readonly (10255) and cadvisor (4194) ports are closed on the public IP address
+	disabledPorts := []int{ports.KubeletReadOnlyPort, 4194}
+	for _, port := range disabledPorts {
+		It(fmt.Sprintf("should not have port %d open on its all public IP addresses", port), func() {
+			portClosedTest(f, node, port)
+		})
+	}
+})
+
+// checks whether the target port is closed
+func portClosedTest(f *framework.Framework, pickNode *v1.Node, port int) {
+	nodeAddrs := framework.GetNodeAddresses(pickNode, v1.NodeExternalIP)
+	Expect(len(nodeAddrs)).NotTo(BeZero())
+
+	for _, addr := range nodeAddrs {
+		conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", addr, port), 1*time.Minute)
+		if err == nil {
+			conn.Close()
+			framework.Failf("port %d is not disabled", port)
+		}
+	}
+}