PodSecurityPolicy E2E tests

cluster/gce/config-test.sh

@@ -298,8 +298,16 @@ if [[ -n "${GCE_GLBC_IMAGE:-}" ]]; then
   PROVIDER_VARS="${PROVIDER_VARS:-} GCE_GLBC_IMAGE"
 fi
 
-# If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely.
-ADMISSION_CONTROL="${KUBE_ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,ResourceQuota,GenericAdmissionWebhook}"
+if [[ -z "${KUBE_ADMISSION_CONTROL:-}" ]]; then
+  ADMISSION_CONTROL="Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority"
+  if [[ "${ENABLE_POD_SECURITY_POLICY:-}" == "true" ]]; then
+    ADMISSION_CONTROL="${ADMISSION_CONTROL},PodSecurityPolicy"
+  fi
+  # ResourceQuota must come last, or a creation is recorded, but the pod may be forbidden.
+  ADMISSION_CONTROL="${ADMISSION_CONTROL},ResourceQuota,GenericAdmissionWebhook"
+else
+  ADMISSION_CONTROL=${KUBE_ADMISSION_CONTROL}
+fi
 
 # Optional: if set to true kube-up will automatically check for existing resources and clean them up.
 KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false}

test/e2e/auth/BUILD

@@ -13,12 +13,17 @@ go_library(
         "framework.go",
         "metadata_concealment.go",
         "node_authz.go",
+        "pod_security_policy.go",
         "service_accounts.go",
     ],
     importpath = "k8s.io/kubernetes/test/e2e/auth",
     deps = [
+        "//pkg/security/apparmor:go_default_library",
+        "//pkg/security/podsecuritypolicy/seccomp:go_default_library",
+        "//pkg/security/podsecuritypolicy/util:go_default_library",
         "//pkg/util/version:go_default_library",
         "//plugin/pkg/admission/serviceaccount:go_default_library",
+        "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/utils/image:go_default_library",
         "//vendor/github.com/evanphx/json-patch:go_default_library",
@@ -28,15 +33,18 @@ go_library(
         "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/api/rbac/v1beta1:go_default_library",
         "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
         "//vendor/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//vendor/k8s.io/apiextensions-apiserver/test/integration/testserver:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",

test/e2e/common/apparmor.go

@@ -40,6 +40,10 @@ const (
 // AppArmorDistros are distros with AppArmor support
 var AppArmorDistros = []string{"gci", "ubuntu"}
 
+func IsAppArmorSupported() bool {
+	return framework.NodeOSDistroIs(AppArmorDistros...)
+}
+
 func SkipIfAppArmorNotSupported() {
 	framework.SkipUnlessNodeOSDistroIs(AppArmorDistros...)
 }

test/e2e/framework/BUILD

@@ -26,6 +26,7 @@ go_library(
         "nodes_util.go",
         "perf_util.go",
         "pods.go",
+        "psp_util.go",
         "pv_util.go",
         "rc_util.go",
         "resource_usage_gatherer.go",
@@ -66,6 +67,7 @@ go_library(
         "//pkg/kubelet/util/format:go_default_library",
         "//pkg/kubemark:go_default_library",
         "//pkg/master/ports:go_default_library",
+        "//pkg/security/podsecuritypolicy/seccomp:go_default_library",
         "//pkg/ssh:go_default_library",
         "//pkg/util/file:go_default_library",
         "//pkg/util/system:go_default_library",
@@ -125,6 +127,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library",
         "//vendor/k8s.io/client-go/discovery:go_default_library",
         "//vendor/k8s.io/client-go/dynamic:go_default_library",
         "//vendor/k8s.io/client-go/informers:go_default_library",

test/e2e/framework/authorizer_util.go

@@ -18,6 +18,7 @@ package framework
 
 import (
 	"fmt"
+	"sync"
 	"time"
 
 	authorizationv1beta1 "k8s.io/api/authorization/v1beta1"
@@ -38,6 +39,12 @@ const (
 // WaitForAuthorizationUpdate checks if the given user can perform the named verb and action.
 // If policyCachePollTimeout is reached without the expected condition matching, an error is returned
 func WaitForAuthorizationUpdate(c v1beta1authorization.SubjectAccessReviewsGetter, user, namespace, verb string, resource schema.GroupResource, allowed bool) error {
+	return WaitForNamedAuthorizationUpdate(c, user, namespace, verb, "", resource, allowed)
+}
+
+// WaitForAuthorizationUpdate checks if the given user can perform the named verb and action on the named resource.
+// If policyCachePollTimeout is reached without the expected condition matching, an error is returned
+func WaitForNamedAuthorizationUpdate(c v1beta1authorization.SubjectAccessReviewsGetter, user, namespace, verb, resourceName string, resource schema.GroupResource, allowed bool) error {
 	review := &authorizationv1beta1.SubjectAccessReview{
 		Spec: authorizationv1beta1.SubjectAccessReviewSpec{
 			ResourceAttributes: &authorizationv1beta1.ResourceAttributes{
@@ -45,6 +52,7 @@ func WaitForAuthorizationUpdate(c v1beta1authorization.SubjectAccessReviewsGette
 				Verb:      verb,
 				Resource:  resource.Resource,
 				Namespace: namespace,
+				Name:      resourceName,
 			},
 			User: user,
 		},
@@ -92,21 +100,52 @@ func BindClusterRole(c v1beta1rbac.ClusterRoleBindingsGetter, clusterRole, ns st
 
 // BindClusterRoleInNamespace binds the cluster role at the namespace scope
 func BindClusterRoleInNamespace(c v1beta1rbac.RoleBindingsGetter, clusterRole, ns string, subjects ...rbacv1beta1.Subject) {
+	bindInNamespace(c, "ClusterRole", clusterRole, ns, subjects...)
+}
+
+// BindRoleInNamespace binds the role at the namespace scope
+func BindRoleInNamespace(c v1beta1rbac.RoleBindingsGetter, role, ns string, subjects ...rbacv1beta1.Subject) {
+	bindInNamespace(c, "Role", role, ns, subjects...)
+}
+
+func bindInNamespace(c v1beta1rbac.RoleBindingsGetter, roleType, role, ns string, subjects ...rbacv1beta1.Subject) {
 	// Since the namespace names are unique, we can leave this lying around so we don't have to race any caches
 	_, err := c.RoleBindings(ns).Create(&rbacv1beta1.RoleBinding{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: ns + "--" + clusterRole,
+			Name: ns + "--" + role,
 		},
 		RoleRef: rbacv1beta1.RoleRef{
 			APIGroup: "rbac.authorization.k8s.io",
-			Kind:     "ClusterRole",
-			Name:     clusterRole,
+			Kind:     roleType,
+			Name:     role,
 		},
 		Subjects: subjects,
 	})
 
 	// if we failed, don't fail the entire test because it may still work. RBAC may simply be disabled.
 	if err != nil {
-		fmt.Printf("Error binding clusterrole/%s into %q for %v\n", clusterRole, ns, subjects)
+		fmt.Printf("Error binding %s/%s into %q for %v\n", roleType, role, ns, subjects)
+	}
+}
+
+var (
+	isRBACEnabledOnce sync.Once
+	isRBACEnabled     bool
+)
+
+func IsRBACEnabled(f *Framework) bool {
+	isRBACEnabledOnce.Do(func() {
+		crs, err := f.ClientSet.RbacV1().ClusterRoles().List(metav1.ListOptions{})
+		if err != nil {
+			Logf("Error listing ClusterRoles; assuming RBAC is disabled: %v", err)
+			isRBACEnabled = false
+		} else if crs == nil || len(crs.Items) == 0 {
+			Logf("No ClusteRoles found; assuming RBAC is disabled.")
+			isRBACEnabled = false
+		} else {
+			Logf("Found ClusterRoles; assuming RBAC is enabled.")
+			isRBACEnabled = true
 		}
+	})
+	return isRBACEnabled
 }

test/e2e/framework/framework.go

@@ -71,6 +71,7 @@ type Framework struct {
 	Namespace                *v1.Namespace   // Every test has at least one namespace unless creation is skipped
 	namespacesToDelete       []*v1.Namespace // Some tests have more than one.
 	NamespaceDeletionTimeout time.Duration
+	SkipPrivilegedPSPBinding bool // Whether to skip creating a binding to the privileged PSP in the test namespace
 
 	gatherer *containerResourceGatherer
 	// Constraints that passed to a check which is executed after data is gathered to
@@ -373,6 +374,11 @@ func (f *Framework) CreateNamespace(baseName string, labels map[string]string) (
 	if ns != nil {
 		f.namespacesToDelete = append(f.namespacesToDelete, ns)
 	}
+
+	if !f.SkipPrivilegedPSPBinding {
+		CreatePrivilegedPSPBinding(f, ns.Name)
+	}
+
 	return ns, err
 }
 

test/e2e/framework/psp_util.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"sync"
+
+	corev1 "k8s.io/api/core/v1"
+	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
+	rbacv1beta1 "k8s.io/api/rbac/v1beta1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/authentication/serviceaccount"
+	"k8s.io/kubernetes/pkg/security/podsecuritypolicy/seccomp"
+
+	. "github.com/onsi/ginkgo"
+)
+
+const (
+	podSecurityPolicyPrivileged = "e2e-test-privileged-psp"
+)
+
+var (
+	isPSPEnabledOnce sync.Once
+	isPSPEnabled     bool
+)
+
+// Creates a PodSecurityPolicy that allows everything.
+func PrivilegedPSP(name string) *extensionsv1beta1.PodSecurityPolicy {
+	allowPrivilegeEscalation := true
+	return &extensionsv1beta1.PodSecurityPolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        name,
+			Annotations: map[string]string{seccomp.AllowedProfilesAnnotationKey: seccomp.AllowAny},
+		},
+		Spec: extensionsv1beta1.PodSecurityPolicySpec{
+			Privileged:               true,
+			AllowPrivilegeEscalation: &allowPrivilegeEscalation,
+			AllowedCapabilities:      []corev1.Capability{"*"},
+			Volumes:                  []extensionsv1beta1.FSType{extensionsv1beta1.All},
+			HostNetwork:              true,
+			HostPorts:                []extensionsv1beta1.HostPortRange{{Min: 0, Max: 65535}},
+			HostIPC:                  true,
+			HostPID:                  true,
+			RunAsUser: extensionsv1beta1.RunAsUserStrategyOptions{
+				Rule: extensionsv1beta1.RunAsUserStrategyRunAsAny,
+			},
+			SELinux: extensionsv1beta1.SELinuxStrategyOptions{
+				Rule: extensionsv1beta1.SELinuxStrategyRunAsAny,
+			},
+			SupplementalGroups: extensionsv1beta1.SupplementalGroupsStrategyOptions{
+				Rule: extensionsv1beta1.SupplementalGroupsStrategyRunAsAny,
+			},
+			FSGroup: extensionsv1beta1.FSGroupStrategyOptions{
+				Rule: extensionsv1beta1.FSGroupStrategyRunAsAny,
+			},
+			ReadOnlyRootFilesystem: false,
+		},
+	}
+}
+
+func IsPodSecurityPolicyEnabled(f *Framework) bool {
+	isPSPEnabledOnce.Do(func() {
+		psps, err := f.ClientSet.ExtensionsV1beta1().PodSecurityPolicies().List(metav1.ListOptions{})
+		if err != nil {
+			Logf("Error listing PodSecurityPolicies; assuming PodSecurityPolicy is disabled: %v", err)
+			isPSPEnabled = false
+		} else if psps == nil || len(psps.Items) == 0 {
+			Logf("No PodSecurityPolicies found; assuming PodSecurityPolicy is disabled.")
+			isPSPEnabled = false
+		} else {
+			Logf("Found PodSecurityPolicies; assuming PodSecurityPolicy is enabled.")
+			isPSPEnabled = true
+		}
+	})
+	return isPSPEnabled
+}
+
+var (
+	privilegedPSPOnce sync.Once
+)
+
+func CreatePrivilegedPSPBinding(f *Framework, namespace string) {
+	if !IsPodSecurityPolicyEnabled(f) {
+		return
+	}
+	// Create the privileged PSP & role
+	privilegedPSPOnce.Do(func() {
+		_, err := f.ClientSet.ExtensionsV1beta1().PodSecurityPolicies().Get(
+			podSecurityPolicyPrivileged, metav1.GetOptions{})
+		if !apierrs.IsNotFound(err) {
+			// Privileged PSP was already created.
+			ExpectNoError(err, "Failed to get PodSecurityPolicy %s", podSecurityPolicyPrivileged)
+			return
+		}
+
+		psp := PrivilegedPSP(podSecurityPolicyPrivileged)
+		psp, err = f.ClientSet.ExtensionsV1beta1().PodSecurityPolicies().Create(psp)
+		ExpectNoError(err, "Failed to create PSP %s", podSecurityPolicyPrivileged)
+
+		// Create the Role to bind it to the namespace.
+		_, err = f.ClientSet.RbacV1beta1().ClusterRoles().Create(&rbacv1beta1.ClusterRole{
+			ObjectMeta: metav1.ObjectMeta{Name: podSecurityPolicyPrivileged},
+			Rules: []rbacv1beta1.PolicyRule{{
+				APIGroups:     []string{"extensions"},
+				Resources:     []string{"podsecuritypolicies"},
+				ResourceNames: []string{podSecurityPolicyPrivileged},
+				Verbs:         []string{"use"},
+			}},
+		})
+		ExpectNoError(err, "Failed to create PSP role")
+	})
+
+	By(fmt.Sprintf("Binding the %s PodSecurityPolicy to the default service account in %s",
+		podSecurityPolicyPrivileged, namespace))
+	BindClusterRoleInNamespace(f.ClientSet.RbacV1beta1(),
+		podSecurityPolicyPrivileged,
+		namespace,
+		rbacv1beta1.Subject{
+			Kind:      rbacv1beta1.ServiceAccountKind,
+			Namespace: namespace,
+			Name:      "default",
+		})
+	ExpectNoError(WaitForNamedAuthorizationUpdate(f.ClientSet.AuthorizationV1beta1(),
+		serviceaccount.MakeUsername(namespace, "default"), namespace, "use", podSecurityPolicyPrivileged,
+		schema.GroupResource{Group: "extensions", Resource: "podsecuritypolicies"}, true))
+}