CRI: move apparmor annotations to container security context

pkg/kubelet/api/v1alpha1/runtime/api.proto

@@ -269,17 +269,7 @@ message PodSandboxConfig {
     // and the CRI). Whenever possible, however, runtime authors SHOULD
     // consider proposing new typed fields for any new features instead.
     //
-    // 1. AppArmor
-    //
-    //    key: container.apparmor.security.beta.kubernetes.io/<container_name>
-    //    description: apparmor profile for a container in this pod.
-    //    value:
-    //      * runtime/default: equivalent to not specifying a profile.
-    //      * localhost/<profile_name>: profile loaded on the node
-    //        (localhost) by name. The possible profile names are detailed at
-    //        http://wiki.apparmor.net/index.php/AppArmor_Core_Policy_Reference
-    //
-    // 2. Seccomp
+    // 1. Seccomp
     //
     //      key: security.alpha.kubernetes.io/seccomp/pod
     //      description: the seccomp profile for the containers of an entire pod.
@@ -296,7 +286,7 @@ message PodSandboxConfig {
     //        local seccomp profile root. Note that profile root is set in
     //        kubelet, and it is not passed in CRI yet, see https://issues.k8s.io/36997.
     //
-    // 3. Sysctls
+    // 2. Sysctls
     //
     //      key: security.alpha.kubernetes.io/sysctls
     //      description: list of safe sysctls which are set for the sandbox.
@@ -526,6 +516,12 @@ message LinuxContainerSecurityContext {
     // List of groups applied to the first process run in the container, in
     // addition to the container's primary GID.
     repeated int64 supplemental_groups = 8;
+    // AppArmor profile for the container, candidate values are:
+    // * runtime/default: equivalent to not specifying a profile.
+    // * localhost/<profile_name>: profile loaded on the node
+    //    (localhost) by name. The possible profile names are detailed at
+    //    http://wiki.apparmor.net/index.php/AppArmor_Core_Policy_Reference
+    string apparmor_profile = 9;
 }
 
 // LinuxContainerConfig contains platform-specific configuration for

pkg/kubelet/dockershim/docker_container.go

@@ -145,7 +145,6 @@ func (ds *dockerService) CreateContainer(podSandboxID string, config *runtimeapi
 
 	// Apply Linux-specific options if applicable.
 	if lc := config.GetLinux(); lc != nil {
-		// Apply resource options.
 		// TODO: Check if the units are correct.
 		// TODO: Can we assume the defaults are sane?
 		rOpts := lc.GetResources()
@@ -162,7 +161,9 @@ func (ds *dockerService) CreateContainer(podSandboxID string, config *runtimeapi
 		// Note: ShmSize is handled in kube_docker_client.go
 
 		// Apply security context.
-		applyContainerSecurityContext(lc, podSandboxID, createConfig.Config, hc, securityOptSep)
+		if err = applyContainerSecurityContext(lc, podSandboxID, createConfig.Config, hc, securityOptSep); err != nil {
+			return "", fmt.Errorf("failed to apply container security context for container %q: %v", config.Metadata.Name, err)
+		}
 		modifyPIDNamespaceOverrides(ds.disableSharedPID, apiVersion, hc)
 	}
 
@@ -187,12 +188,12 @@ func (ds *dockerService) CreateContainer(podSandboxID string, config *runtimeapi
 	}
 	hc.Resources.Devices = devices
 
-	// Apply appArmor and seccomp options.
-	securityOpts, err := getContainerSecurityOpts(config.Metadata.Name, sandboxConfig, ds.seccompProfileRoot, securityOptSep)
+	// Apply seccomp options.
+	seccompSecurityOpts, err := getSeccompSecurityOpts(config.Metadata.Name, sandboxConfig, ds.seccompProfileRoot, securityOptSep)
 	if err != nil {
-		return "", fmt.Errorf("failed to generate container security options for container %q: %v", config.Metadata.Name, err)
+		return "", fmt.Errorf("failed to generate seccomp security options for container %q: %v", config.Metadata.Name, err)
 	}
-	hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...)
+	hc.SecurityOpt = append(hc.SecurityOpt, seccompSecurityOpts...)
 
 	createConfig.HostConfig = hc
 	createResp, err := ds.client.CreateContainer(createConfig)

pkg/kubelet/dockershim/docker_sandbox.go

@@ -475,7 +475,9 @@ func (ds *dockerService) applySandboxLinuxOptions(hc *dockercontainer.HostConfig
 	}
 	hc.CgroupParent = cgroupParent
 	// Apply security context.
-	applySandboxSecurityContext(lc, createConfig.Config, hc, ds.network, separator)
+	if err = applySandboxSecurityContext(lc, createConfig.Config, hc, ds.network, separator); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -541,7 +543,7 @@ func (ds *dockerService) makeSandboxDockerConfig(c *runtimeapi.PodSandboxConfig,
 	}
 
 	// Set security options.
-	securityOpts, err := getSandboxSecurityOpts(c, ds.seccompProfileRoot, securityOptSep)
+	securityOpts, err := getSeccompSecurityOpts(sandboxContainerName, c, ds.seccompProfileRoot, securityOptSep)
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate sandbox security options for sandbox %q: %v", c.Metadata.Name, err)
 	}

pkg/kubelet/dockershim/helpers.go

@@ -181,26 +181,32 @@ func makePortsAndBindings(pm []*runtimeapi.PortMapping) (map[dockernat.Port]stru
 	return exposedPorts, portBindings
 }
 
-// getContainerSecurityOpt gets container security options from container and sandbox config, currently from sandbox
-// annotations.
+// getSeccompSecurityOpts gets container seccomp options from container and sandbox
+// config, currently from sandbox annotations.
 // It is an experimental feature and may be promoted to official runtime api in the future.
-func getContainerSecurityOpts(containerName string, sandboxConfig *runtimeapi.PodSandboxConfig, seccompProfileRoot string, separator rune) ([]string, error) {
-	appArmorOpts, err := dockertools.GetAppArmorOpts(sandboxConfig.GetAnnotations(), containerName)
-	if err != nil {
-		return nil, err
-	}
+func getSeccompSecurityOpts(containerName string, sandboxConfig *runtimeapi.PodSandboxConfig, seccompProfileRoot string, separator rune) ([]string, error) {
 	seccompOpts, err := dockertools.GetSeccompOpts(sandboxConfig.GetAnnotations(), containerName, seccompProfileRoot)
 	if err != nil {
 		return nil, err
 	}
-	securityOpts := append(appArmorOpts, seccompOpts...)
-	fmtOpts := dockertools.FmtDockerOpts(securityOpts, separator)
+
+	fmtOpts := dockertools.FmtDockerOpts(seccompOpts, separator)
 	return fmtOpts, nil
 }
 
-func getSandboxSecurityOpts(sandboxConfig *runtimeapi.PodSandboxConfig, seccompProfileRoot string, separator rune) ([]string, error) {
-	// sandboxContainerName doesn't exist in the pod, so pod security options will be returned by default.
-	return getContainerSecurityOpts(sandboxContainerName, sandboxConfig, seccompProfileRoot, separator)
+// getApparmorSecurityOpts gets apparmor options from container config.
+func getApparmorSecurityOpts(sc *runtimeapi.LinuxContainerSecurityContext, separator rune) ([]string, error) {
+	if sc == nil || sc.ApparmorProfile == "" {
+		return nil, nil
+	}
+
+	appArmorOpts, err := dockertools.GetAppArmorOpts(sc.ApparmorProfile)
+	if err != nil {
+		return nil, err
+	}
+
+	fmtOpts := dockertools.FmtDockerOpts(appArmorOpts, separator)
+	return fmtOpts, nil
 }
 
 func getNetworkNamespace(c *dockertypes.ContainerJSON) string {

pkg/kubelet/dockershim/helpers_test.go

@@ -43,10 +43,10 @@ func TestLabelsAndAnnotationsRoundTrip(t *testing.T) {
 	assert.Equal(t, expectedAnnotations, actualAnnotations)
 }
 
-// TestGetContainerSecurityOpts tests the logic of generating container security options from sandbox annotations.
+// TestGetSeccompSecurityOpts tests the logic of generating container seccomp options from sandbox annotations.
 // The actual profile loading logic is tested in dockertools.
 // TODO: Migrate the corresponding test to dockershim.
-func TestGetContainerSecurityOpts(t *testing.T) {
+func TestGetSeccompSecurityOpts(t *testing.T) {
 	containerName := "bar"
 	makeConfig := func(annotations map[string]string) *runtimeapi.PodSandboxConfig {
 		return makeSandboxConfigWithLabelsAndAnnotations("pod", "ns", "1234", 1, nil, annotations)
@@ -78,29 +78,10 @@ func TestGetContainerSecurityOpts(t *testing.T) {
 			v1.SeccompPodAnnotationKey: "docker/default",
 		}),
 		expectedOpts: nil,
-	}, {
-		msg: "AppArmor runtime/default",
-		config: makeConfig(map[string]string{
-			apparmor.ContainerAnnotationKeyPrefix + containerName: apparmor.ProfileRuntimeDefault,
-		}),
-		expectedOpts: []string{"seccomp=unconfined"},
-	}, {
-		msg: "AppArmor local profile",
-		config: makeConfig(map[string]string{
-			apparmor.ContainerAnnotationKeyPrefix + containerName: apparmor.ProfileNamePrefix + "foo",
-		}),
-		expectedOpts: []string{"seccomp=unconfined", "apparmor=foo"},
-	}, {
-		msg: "AppArmor and seccomp profile",
-		config: makeConfig(map[string]string{
-			v1.SeccompContainerAnnotationKeyPrefix + containerName: "docker/default",
-			apparmor.ContainerAnnotationKeyPrefix + containerName:  apparmor.ProfileNamePrefix + "foo",
-		}),
-		expectedOpts: []string{"apparmor=foo"},
 	}}
 
 	for i, test := range tests {
-		opts, err := getContainerSecurityOpts(containerName, test.config, "test/seccomp/profile/root", '=')
+		opts, err := getSeccompSecurityOpts(containerName, test.config, "test/seccomp/profile/root", '=')
 		assert.NoError(t, err, "TestCase[%d]: %s", i, test.msg)
 		assert.Len(t, opts, len(test.expectedOpts), "TestCase[%d]: %s", i, test.msg)
 		for _, opt := range test.expectedOpts {
@@ -109,43 +90,36 @@ func TestGetContainerSecurityOpts(t *testing.T) {
 	}
 }
 
-// TestGetSandboxSecurityOpts tests the logic of generating sandbox security options from sandbox annotations.
-func TestGetSandboxSecurityOpts(t *testing.T) {
-	makeConfig := func(annotations map[string]string) *runtimeapi.PodSandboxConfig {
-		return makeSandboxConfigWithLabelsAndAnnotations("pod", "ns", "1234", 1, nil, annotations)
+// TestGetApparmorSecurityOpts tests the logic of generating container apparmor options from sandbox annotations.
+// The actual profile loading logic is tested in dockertools.
+// TODO: Migrate the corresponding test to dockershim.
+func TestGetApparmorSecurityOpts(t *testing.T) {
+	makeConfig := func(profile string) *runtimeapi.LinuxContainerSecurityContext {
+		return &runtimeapi.LinuxContainerSecurityContext{
+			ApparmorProfile: profile,
+		}
 	}
 
 	tests := []struct {
 		msg          string
-		config       *runtimeapi.PodSandboxConfig
+		config       *runtimeapi.LinuxContainerSecurityContext
 		expectedOpts []string
 	}{{
-		msg:          "No security annotations",
-		config:       makeConfig(nil),
-		expectedOpts: []string{"seccomp=unconfined"},
-	}, {
-		msg: "Seccomp default",
-		config: makeConfig(map[string]string{
-			v1.SeccompPodAnnotationKey: "docker/default",
-		}),
+		msg:          "No AppArmor options",
+		config:       makeConfig(""),
 		expectedOpts: nil,
 	}, {
-		msg: "Seccomp unconfined",
-		config: makeConfig(map[string]string{
-			v1.SeccompPodAnnotationKey: "unconfined",
-		}),
-		expectedOpts: []string{"seccomp=unconfined"},
+		msg:          "AppArmor runtime/default",
+		config:       makeConfig("runtime/default"),
+		expectedOpts: []string{},
 	}, {
-		msg: "Seccomp pod and container profile",
-		config: makeConfig(map[string]string{
-			v1.SeccompContainerAnnotationKeyPrefix + "test-container": "unconfined",
-			v1.SeccompPodAnnotationKey:                                "docker/default",
-		}),
-		expectedOpts: nil,
+		msg:          "AppArmor local profile",
+		config:       makeConfig(apparmor.ProfileNamePrefix + "foo"),
+		expectedOpts: []string{"apparmor=foo"},
 	}}
 
 	for i, test := range tests {
-		opts, err := getSandboxSecurityOpts(test.config, "test/seccomp/profile/root", '=')
+		opts, err := getApparmorSecurityOpts(test.config, '=')
 		assert.NoError(t, err, "TestCase[%d]: %s", i, test.msg)
 		assert.Len(t, opts, len(test.expectedOpts), "TestCase[%d]: %s", i, test.msg)
 		for _, opt := range test.expectedOpts {

pkg/kubelet/dockershim/security_context.go

@@ -31,9 +31,9 @@ import (
 )
 
 // applySandboxSecurityContext updates docker sandbox options according to security context.
-func applySandboxSecurityContext(lc *runtimeapi.LinuxPodSandboxConfig, config *dockercontainer.Config, hc *dockercontainer.HostConfig, network *knetwork.PluginManager, separator rune) {
+func applySandboxSecurityContext(lc *runtimeapi.LinuxPodSandboxConfig, config *dockercontainer.Config, hc *dockercontainer.HostConfig, network *knetwork.PluginManager, separator rune) error {
 	if lc == nil {
-		return
+		return nil
 	}
 
 	var sc *runtimeapi.LinuxContainerSecurityContext
@@ -48,20 +48,25 @@ func applySandboxSecurityContext(lc *runtimeapi.LinuxPodSandboxConfig, config *d
 	}
 
 	modifyContainerConfig(sc, config)
-	modifyHostConfig(sc, hc, separator)
+	if err := modifyHostConfig(sc, hc, separator); err != nil {
+		return err
+	}
 	modifySandboxNamespaceOptions(sc.GetNamespaceOptions(), hc, network)
+	return nil
 }
 
 // applyContainerSecurityContext updates docker container options according to security context.
-func applyContainerSecurityContext(lc *runtimeapi.LinuxContainerConfig, sandboxID string, config *dockercontainer.Config, hc *dockercontainer.HostConfig, separator rune) {
+func applyContainerSecurityContext(lc *runtimeapi.LinuxContainerConfig, sandboxID string, config *dockercontainer.Config, hc *dockercontainer.HostConfig, separator rune) error {
 	if lc == nil {
-		return
+		return nil
 	}
 
 	modifyContainerConfig(lc.SecurityContext, config)
-	modifyHostConfig(lc.SecurityContext, hc, separator)
+	if err := modifyHostConfig(lc.SecurityContext, hc, separator); err != nil {
+		return err
+	}
 	modifyContainerNamespaceOptions(lc.SecurityContext.GetNamespaceOptions(), sandboxID, hc)
-	return
+	return nil
 }
 
 // modifyContainerConfig applies container security context config to dockercontainer.Config.
@@ -78,9 +83,9 @@ func modifyContainerConfig(sc *runtimeapi.LinuxContainerSecurityContext, config
 }
 
 // modifyHostConfig applies security context config to dockercontainer.HostConfig.
-func modifyHostConfig(sc *runtimeapi.LinuxContainerSecurityContext, hostConfig *dockercontainer.HostConfig, separator rune) {
+func modifyHostConfig(sc *runtimeapi.LinuxContainerSecurityContext, hostConfig *dockercontainer.HostConfig, separator rune) error {
 	if sc == nil {
-		return
+		return nil
 	}
 
 	// Apply supplemental groups.
@@ -107,6 +112,15 @@ func modifyHostConfig(sc *runtimeapi.LinuxContainerSecurityContext, hostConfig *
 			separator,
 		)
 	}
+
+	// Apply apparmor options.
+	apparmorSecurityOpts, err := getApparmorSecurityOpts(sc, separator)
+	if err != nil {
+		return fmt.Errorf("failed to generate apparmor security options: %v", err)
+	}
+	hostConfig.SecurityOpt = append(hostConfig.SecurityOpt, apparmorSecurityOpts...)
+
+	return nil
 }
 
 // modifySandboxNamespaceOptions apply namespace options for sandbox

pkg/kubelet/dockertools/docker_manager.go

@@ -1260,13 +1260,13 @@ func GetSeccompOpts(annotations map[string]string, ctrName, profileRoot string)
 
 // Get the docker security options for AppArmor.
 func (dm *DockerManager) getAppArmorOpts(pod *v1.Pod, ctrName string) ([]dockerOpt, error) {
-	return GetAppArmorOpts(pod.Annotations, ctrName)
+	profile := apparmor.GetProfileNameFromPodAnnotations(pod.Annotations, ctrName)
+	return GetAppArmorOpts(profile)
 }
 
 // Temporarily export this function to share with dockershim.
 // TODO: clean this up.
-func GetAppArmorOpts(annotations map[string]string, ctrName string) ([]dockerOpt, error) {
-	profile := apparmor.GetProfileNameFromPodAnnotations(annotations, ctrName)
+func GetAppArmorOpts(profile string) ([]dockerOpt, error) {
 	if profile == "" || profile == apparmor.ProfileRuntimeDefault {
 		// The docker applies the default profile by default.
 		return nil, nil

pkg/kubelet/kuberuntime/BUILD

@@ -45,6 +45,7 @@ go_library(
         "//pkg/kubelet/types:go_default_library",
         "//pkg/kubelet/util/cache:go_default_library",
         "//pkg/kubelet/util/format:go_default_library",
+        "//pkg/security/apparmor:go_default_library",
         "//pkg/securitycontext:go_default_library",
         "//pkg/util/parsers:go_default_library",
         "//pkg/util/selinux:go_default_library",

pkg/kubelet/kuberuntime/security_context.go

@@ -21,6 +21,7 @@ import (
 
 	"k8s.io/kubernetes/pkg/api/v1"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
+	"k8s.io/kubernetes/pkg/security/apparmor"
 	"k8s.io/kubernetes/pkg/securitycontext"
 )
 
@@ -32,6 +33,9 @@ func (m *kubeGenericRuntimeManager) determineEffectiveSecurityContext(pod *v1.Po
 		synthesized = &runtimeapi.LinuxContainerSecurityContext{}
 	}
 
+	// set ApparmorProfile.
+	synthesized.ApparmorProfile = apparmor.GetProfileNameFromPodAnnotations(pod.Annotations, container.Name)
+
 	// set RunAsUser.
 	if synthesized.RunAsUser == nil {
 		if uid != nil {