Merge pull request #75459 from wk8/wk8/gmsa_beta

api/openapi-spec/swagger.json

@@ -11222,6 +11222,16 @@
     },
     "io.k8s.api.core.v1.WindowsSecurityContextOptions": {
       "description": "WindowsSecurityContextOptions contain Windows-specific options and credentials.",
+      "properties": {
+        "gmsaCredentialSpec": {
+          "description": "GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.",
+          "type": "string"
+        },
+        "gmsaCredentialSpecName": {
+          "description": "GMSACredentialSpecName is the name of the GMSA credential spec to use. This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.",
+          "type": "string"
+        }
+      },
       "type": "object"
     },
     "io.k8s.api.events.v1beta1.Event": {

pkg/api/pod/util.go

@@ -368,6 +368,8 @@ func dropDisabledFields(
 
 	dropDisabledRunAsGroupField(podSpec, oldPodSpec)
 
+	dropDisabledGMSAFields(podSpec, oldPodSpec)
+
 	if !utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && !runtimeClassInUse(oldPodSpec) {
 		// Set RuntimeClassName to nil only if feature is disabled and it is not used
 		podSpec.RuntimeClassName = nil
@@ -399,6 +401,39 @@ func dropDisabledRunAsGroupField(podSpec, oldPodSpec *api.PodSpec) {
 	}
 }
 
+// dropDisabledGMSAFields removes disabled fields related to Windows GMSA
+// from the given PodSpec.
+func dropDisabledGMSAFields(podSpec, oldPodSpec *api.PodSpec) {
+	if utilfeature.DefaultFeatureGate.Enabled(features.WindowsGMSA) ||
+		gMSAFieldsInUse(oldPodSpec) {
+		return
+	}
+
+	if podSpec.SecurityContext != nil {
+		dropDisabledGMSAFieldsFromWindowsSecurityOptions(podSpec.SecurityContext.WindowsOptions)
+	}
+	dropDisabledGMSAFieldsFromContainers(podSpec.Containers)
+	dropDisabledGMSAFieldsFromContainers(podSpec.InitContainers)
+}
+
+// dropDisabledGMSAFieldsFromWindowsSecurityOptions removes disabled fields
+// related to Windows GMSA from the given WindowsSecurityContextOptions.
+func dropDisabledGMSAFieldsFromWindowsSecurityOptions(windowsOptions *api.WindowsSecurityContextOptions) {
+	if windowsOptions != nil {
+		windowsOptions.GMSACredentialSpecName = nil
+		windowsOptions.GMSACredentialSpec = nil
+	}
+}
+
+// dropDisabledGMSAFieldsFromContainers removes disabled fields
+func dropDisabledGMSAFieldsFromContainers(containers []api.Container) {
+	for i := range containers {
+		if containers[i].SecurityContext != nil {
+			dropDisabledGMSAFieldsFromWindowsSecurityOptions(containers[i].SecurityContext.WindowsOptions)
+		}
+	}
+}
+
 // dropDisabledProcMountField removes disabled fields from PodSpec related
 // to ProcMount only if it is not already used by the old spec
 func dropDisabledProcMountField(podSpec, oldPodSpec *api.PodSpec) {
@@ -612,6 +647,44 @@ func runAsGroupInUse(podSpec *api.PodSpec) bool {
 	return false
 }
 
+// gMSAFieldsInUse returns true if the pod spec is non-nil and has one of any
+// SecurityContext's GMSACredentialSpecName or GMSACredentialSpec fields set.
+func gMSAFieldsInUse(podSpec *api.PodSpec) bool {
+	if podSpec == nil {
+		return false
+	}
+
+	if podSpec.SecurityContext != nil && gMSAFieldsInUseInWindowsSecurityOptions(podSpec.SecurityContext.WindowsOptions) {
+		return true
+	}
+
+	return gMSAFieldsInUseInAnyContainer(podSpec.Containers) ||
+		gMSAFieldsInUseInAnyContainer(podSpec.InitContainers)
+}
+
+// gMSAFieldsInUseInWindowsSecurityOptions returns true if the given WindowsSecurityContextOptions is
+// non-nil and one of its GMSACredentialSpecName or GMSACredentialSpec fields is set.
+func gMSAFieldsInUseInWindowsSecurityOptions(windowsOptions *api.WindowsSecurityContextOptions) bool {
+	if windowsOptions == nil {
+		return false
+	}
+
+	return windowsOptions.GMSACredentialSpecName != nil ||
+		windowsOptions.GMSACredentialSpec != nil
+}
+
+// gMSAFieldsInUseInAnyContainer returns true if any of the given Containers has its
+// SecurityContext's GMSACredentialSpecName or GMSACredentialSpec fields set.
+func gMSAFieldsInUseInAnyContainer(containers []api.Container) bool {
+	for _, container := range containers {
+		if container.SecurityContext != nil && gMSAFieldsInUseInWindowsSecurityOptions(container.SecurityContext.WindowsOptions) {
+			return true
+		}
+	}
+
+	return false
+}
+
 // subpathExprInUse returns true if the pod spec is non-nil and has a volume mount that makes use of the subPathExpr feature
 func subpathExprInUse(podSpec *api.PodSpec) bool {
 	if podSpec == nil {

pkg/api/pod/util_test.go

@@ -1359,6 +1359,208 @@ func TestDropRunAsGroup(t *testing.T) {
 	}
 }
 
+func TestDropGMSAFields(t *testing.T) {
+	defaultContainerSecurityContextFactory := func() *api.SecurityContext {
+		defaultProcMount := api.DefaultProcMount
+		return &api.SecurityContext{ProcMount: &defaultProcMount}
+	}
+	podWithoutWindowsOptionsFactory := func() *api.Pod {
+		return &api.Pod{
+			Spec: api.PodSpec{
+				RestartPolicy:   api.RestartPolicyNever,
+				SecurityContext: &api.PodSecurityContext{},
+				Containers:      []api.Container{{Name: "container1", Image: "testimage", SecurityContext: defaultContainerSecurityContextFactory()}},
+				InitContainers:  []api.Container{{Name: "initContainer1", Image: "testimage", SecurityContext: defaultContainerSecurityContextFactory()}},
+			},
+		}
+	}
+
+	type podFactoryInfo struct {
+		description  string
+		hasGMSAField bool
+		// this factory should generate the input pod whose spec will be fed to dropDisabledFields
+		podFactory func() *api.Pod
+		// this factory should generate the expected pod after the GMSA fields have been dropped
+		// we can't just use podWithoutWindowsOptionsFactory as is for this, since in some cases
+		// we'll be left with a WindowsSecurityContextOptions struct with no GMSA field set, as opposed
+		// to a nil pointer in the pod generated by podWithoutWindowsOptionsFactory
+		// if this field is not set, it will default to the podFactory
+		strippedPodFactory func() *api.Pod
+	}
+	podFactoryInfos := []podFactoryInfo{
+		{
+			description:  "does not have any GMSA field set",
+			hasGMSAField: false,
+			podFactory:   podWithoutWindowsOptionsFactory,
+		},
+		{
+			description:  "has a pod-level WindowsSecurityContextOptions struct with no GMSA field set",
+			hasGMSAField: false,
+			podFactory: func() *api.Pod {
+				pod := podWithoutWindowsOptionsFactory()
+				pod.Spec.SecurityContext.WindowsOptions = &api.WindowsSecurityContextOptions{}
+				return pod
+			},
+		},
+		{
+			description:  "has a WindowsSecurityContextOptions struct with no GMSA field set on a container",
+			hasGMSAField: false,
+			podFactory: func() *api.Pod {
+				pod := podWithoutWindowsOptionsFactory()
+				pod.Spec.Containers[0].SecurityContext.WindowsOptions = &api.WindowsSecurityContextOptions{}
+				return pod
+			},
+		},
+		{
+			description:  "has a WindowsSecurityContextOptions struct with no GMSA field set on an init container",
+			hasGMSAField: false,
+			podFactory: func() *api.Pod {
+				pod := podWithoutWindowsOptionsFactory()
+				pod.Spec.InitContainers[0].SecurityContext.WindowsOptions = &api.WindowsSecurityContextOptions{}
+				return pod
+			},
+		},
+		{
+			description:  "is nil",
+			hasGMSAField: false,
+			podFactory:   func() *api.Pod { return nil },
+		},
+	}
+
+	toPtr := func(s string) *string {
+		return &s
+	}
+	addGMSACredentialSpecName := func(windowsOptions *api.WindowsSecurityContextOptions) {
+		windowsOptions.GMSACredentialSpecName = toPtr("dummy-gmsa-cred-spec-name")
+	}
+	addGMSACredentialSpec := func(windowsOptions *api.WindowsSecurityContextOptions) {
+		windowsOptions.GMSACredentialSpec = toPtr("dummy-gmsa-cred-spec-contents")
+	}
+	addBothGMSAFields := func(windowsOptions *api.WindowsSecurityContextOptions) {
+		addGMSACredentialSpecName(windowsOptions)
+		addGMSACredentialSpec(windowsOptions)
+	}
+
+	for fieldName, windowsOptionsTransformingFunc := range map[string]func(*api.WindowsSecurityContextOptions){
+		"GMSACredentialSpecName field": addGMSACredentialSpecName,
+		"GMSACredentialSpec field":     addGMSACredentialSpec,
+		"both GMSA fields":             addBothGMSAFields,
+	} {
+		// yes, these variables are indeed needed for the closure to work
+		// properly, please do NOT remove them
+		name := fieldName
+		transformingFunc := windowsOptionsTransformingFunc
+
+		windowsOptionsWithGMSAFieldFactory := func() *api.WindowsSecurityContextOptions {
+			windowsOptions := &api.WindowsSecurityContextOptions{}
+			transformingFunc(windowsOptions)
+			return windowsOptions
+		}
+
+		podFactoryInfos = append(podFactoryInfos,
+			podFactoryInfo{
+				description:  fmt.Sprintf("has %s in Pod", name),
+				hasGMSAField: true,
+				podFactory: func() *api.Pod {
+					pod := podWithoutWindowsOptionsFactory()
+					pod.Spec.SecurityContext.WindowsOptions = windowsOptionsWithGMSAFieldFactory()
+					return pod
+				},
+				strippedPodFactory: func() *api.Pod {
+					pod := podWithoutWindowsOptionsFactory()
+					pod.Spec.SecurityContext.WindowsOptions = &api.WindowsSecurityContextOptions{}
+					return pod
+				},
+			},
+			podFactoryInfo{
+				description:  fmt.Sprintf("has %s in Container", name),
+				hasGMSAField: true,
+				podFactory: func() *api.Pod {
+					pod := podWithoutWindowsOptionsFactory()
+					pod.Spec.Containers[0].SecurityContext.WindowsOptions = windowsOptionsWithGMSAFieldFactory()
+					return pod
+				},
+				strippedPodFactory: func() *api.Pod {
+					pod := podWithoutWindowsOptionsFactory()
+					pod.Spec.Containers[0].SecurityContext.WindowsOptions = &api.WindowsSecurityContextOptions{}
+					return pod
+				},
+			},
+			podFactoryInfo{
+				description:  fmt.Sprintf("has %s in InitContainer", name),
+				hasGMSAField: true,
+				podFactory: func() *api.Pod {
+					pod := podWithoutWindowsOptionsFactory()
+					pod.Spec.InitContainers[0].SecurityContext.WindowsOptions = windowsOptionsWithGMSAFieldFactory()
+					return pod
+				},
+				strippedPodFactory: func() *api.Pod {
+					pod := podWithoutWindowsOptionsFactory()
+					pod.Spec.InitContainers[0].SecurityContext.WindowsOptions = &api.WindowsSecurityContextOptions{}
+					return pod
+				},
+			})
+	}
+
+	for _, enabled := range []bool{true, false} {
+		for _, oldPodFactoryInfo := range podFactoryInfos {
+			for _, newPodFactoryInfo := range podFactoryInfos {
+				newPodHasGMSAField, newPod := newPodFactoryInfo.hasGMSAField, newPodFactoryInfo.podFactory()
+				if newPod == nil {
+					continue
+				}
+				oldPodHasGMSAField, oldPod := oldPodFactoryInfo.hasGMSAField, oldPodFactoryInfo.podFactory()
+
+				t.Run(fmt.Sprintf("feature enabled=%v, old pod %s, new pod %s", enabled, oldPodFactoryInfo.description, newPodFactoryInfo.description), func(t *testing.T) {
+					defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WindowsGMSA, enabled)()
+
+					var oldPodSpec *api.PodSpec
+					if oldPod != nil {
+						oldPodSpec = &oldPod.Spec
+					}
+					dropDisabledFields(&newPod.Spec, nil, oldPodSpec, nil)
+
+					// old pod should never be changed
+					if !reflect.DeepEqual(oldPod, oldPodFactoryInfo.podFactory()) {
+						t.Errorf("old pod changed: %v", diff.ObjectReflectDiff(oldPod, oldPodFactoryInfo.podFactory()))
+					}
+
+					switch {
+					case enabled || oldPodHasGMSAField:
+						// new pod should not be changed if the feature is enabled, or if the old pod had any GMSA field set
+						if !reflect.DeepEqual(newPod, newPodFactoryInfo.podFactory()) {
+							t.Errorf("new pod changed: %v", diff.ObjectReflectDiff(newPod, newPodFactoryInfo.podFactory()))
+						}
+					case newPodHasGMSAField:
+						// new pod should be changed
+						if reflect.DeepEqual(newPod, newPodFactoryInfo.podFactory()) {
+							t.Errorf("%v", oldPod)
+							t.Errorf("%v", newPod)
+							t.Errorf("new pod was not changed")
+						}
+						// new pod should not have any GMSA field set
+						var expectedStrippedPod *api.Pod
+						if newPodFactoryInfo.strippedPodFactory == nil {
+							expectedStrippedPod = newPodFactoryInfo.podFactory()
+						} else {
+							expectedStrippedPod = newPodFactoryInfo.strippedPodFactory()
+						}
+
+						if !reflect.DeepEqual(newPod, expectedStrippedPod) {
+							t.Errorf("new pod had some GMSA field set: %v", diff.ObjectReflectDiff(newPod, expectedStrippedPod))
+						}
+					default:
+						// new pod should not need to be changed
+						if !reflect.DeepEqual(newPod, newPodFactoryInfo.podFactory()) {
+							t.Errorf("new pod changed: %v", diff.ObjectReflectDiff(newPod, newPodFactoryInfo.podFactory()))
+						}
+					}
+				})
+			}
+		}
+	}
+}
+
 func TestDropPodSysctls(t *testing.T) {
 	podWithSysctls := func() *api.Pod {
 		return &api.Pod{

pkg/apis/core/types.go

@@ -4739,7 +4739,17 @@ type SELinuxOptions struct {
 
 // WindowsSecurityContextOptions contain Windows-specific options and credentials.
 type WindowsSecurityContextOptions struct {
-	// intentionally left empty for now
+	// GMSACredentialSpecName is the name of the GMSA credential spec to use.
+	// This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.
+	// +optional
+	GMSACredentialSpecName *string
+
+	// GMSACredentialSpec is where the GMSA admission webhook
+	// (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the
+	// GMSA credential spec named by the GMSACredentialSpecName field.
+	// This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.
+	// +optional
+	GMSACredentialSpec *string
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/core/v1/zz_generated.conversion.go

@@ -7599,6 +7599,8 @@ func Convert_core_WeightedPodAffinityTerm_To_v1_WeightedPodAffinityTerm(in *core
 }
 
 func autoConvert_v1_WindowsSecurityContextOptions_To_core_WindowsSecurityContextOptions(in *v1.WindowsSecurityContextOptions, out *core.WindowsSecurityContextOptions, s conversion.Scope) error {
+	out.GMSACredentialSpecName = (*string)(unsafe.Pointer(in.GMSACredentialSpecName))
+	out.GMSACredentialSpec = (*string)(unsafe.Pointer(in.GMSACredentialSpec))
 	return nil
 }
 
@@ -7608,6 +7610,8 @@ func Convert_v1_WindowsSecurityContextOptions_To_core_WindowsSecurityContextOpti
 }
 
 func autoConvert_core_WindowsSecurityContextOptions_To_v1_WindowsSecurityContextOptions(in *core.WindowsSecurityContextOptions, out *v1.WindowsSecurityContextOptions, s conversion.Scope) error {
+	out.GMSACredentialSpecName = (*string)(unsafe.Pointer(in.GMSACredentialSpecName))
+	out.GMSACredentialSpec = (*string)(unsafe.Pointer(in.GMSACredentialSpec))
 	return nil
 }
 

pkg/apis/core/validation/validation.go

@@ -3446,6 +3446,8 @@ func ValidatePodSecurityContext(securityContext *core.PodSecurityContext, spec *
 		if len(securityContext.Sysctls) != 0 {
 			allErrs = append(allErrs, validateSysctls(securityContext.Sysctls, fldPath.Child("sysctls"))...)
 		}
+
+		allErrs = append(allErrs, validateWindowsSecurityContextOptions(securityContext.WindowsOptions, fldPath.Child("windowsOptions"))...)
 	}
 
 	return allErrs
@@ -5156,7 +5158,7 @@ func ValidateEndpointsUpdate(newEndpoints, oldEndpoints *core.Endpoints) field.E
 	return allErrs
 }
 
-// ValidateSecurityContext ensure the security context contains valid settings
+// ValidateSecurityContext ensures the security context contains valid settings
 func ValidateSecurityContext(sc *core.SecurityContext, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 	//this should only be true for testing since SecurityContext is defaulted by the core
@@ -5202,6 +5204,42 @@ func ValidateSecurityContext(sc *core.SecurityContext, fldPath *field.Path) fiel
 		}
 	}
 
+	allErrs = append(allErrs, validateWindowsSecurityContextOptions(sc.WindowsOptions, fldPath.Child("windowsOptions"))...)
+
+	return allErrs
+}
+
+// maxGMSACredentialSpecLength is the max length, in bytes, for the actual contents
+// of a GMSA cred spec. In general, those shouldn't be more than a few hundred bytes,
+// so we want to give plenty of room here while still providing an upper bound.
+const (
+	maxGMSACredentialSpecLengthInKiB = 64
+	maxGMSACredentialSpecLength      = maxGMSACredentialSpecLengthInKiB * 1024
+)
+
+func validateWindowsSecurityContextOptions(windowsOptions *core.WindowsSecurityContextOptions, fieldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if windowsOptions == nil {
+		return allErrs
+	}
+
+	if windowsOptions.GMSACredentialSpecName != nil {
+		// gmsaCredentialSpecName must be the name of a custom resource
+		for _, msg := range validation.IsDNS1123Subdomain(*windowsOptions.GMSACredentialSpecName) {
+			allErrs = append(allErrs, field.Invalid(fieldPath.Child("gmsaCredentialSpecName"), windowsOptions.GMSACredentialSpecName, msg))
+		}
+	}
+
+	if windowsOptions.GMSACredentialSpec != nil {
+		if l := len(*windowsOptions.GMSACredentialSpec); l == 0 {
+			allErrs = append(allErrs, field.Invalid(fieldPath.Child("gmsaCredentialSpec"), windowsOptions.GMSACredentialSpec, "gmsaCredentialSpec cannot be an empty string"))
+		} else if l > maxGMSACredentialSpecLength {
+			errMsg := fmt.Sprintf("gmsaCredentialSpec size must be under %d KiB", maxGMSACredentialSpecLengthInKiB)
+			allErrs = append(allErrs, field.Invalid(fieldPath.Child("gmsaCredentialSpec"), windowsOptions.GMSACredentialSpec, errMsg))
+		}
+	}
+
 	return allErrs
 }
 

pkg/apis/core/validation/validation_test.go

@@ -13205,3 +13205,77 @@ func TestValidateOrSetClientIPAffinityConfig(t *testing.T) {
 		}
 	}
 }
+
+func TestValidateWindowsSecurityContextOptions(t *testing.T) {
+	toPtr := func(s string) *string {
+		return &s
+	}
+
+	testCases := []struct {
+		testName string
+
+		windowsOptions         *core.WindowsSecurityContextOptions
+		expectedErrorSubstring string
+	}{
+		{
+			testName: "a nil pointer",
+		},
+		{
+			testName:       "an empty struct",
+			windowsOptions: &core.WindowsSecurityContextOptions{},
+		},
+		{
+			testName: "a valid input",
+			windowsOptions: &core.WindowsSecurityContextOptions{
+				GMSACredentialSpecName: toPtr("dummy-gmsa-crep-spec-name"),
+				GMSACredentialSpec:     toPtr("dummy-gmsa-crep-spec-contents"),
+			},
+		},
+		{
+			testName: "a GMSA cred spec name that is not a valid resource name",
+			windowsOptions: &core.WindowsSecurityContextOptions{
+				// invalid because of the underscore
+				GMSACredentialSpecName: toPtr("not_a-valid-gmsa-crep-spec-name"),
+			},
+			expectedErrorSubstring: dnsSubdomainLabelErrMsg,
+		},
+		{
+			testName: "empty GMSA cred spec contents",
+			windowsOptions: &core.WindowsSecurityContextOptions{
+				GMSACredentialSpec: toPtr(""),
+			},
+			expectedErrorSubstring: "gmsaCredentialSpec cannot be an empty string",
+		},
+		{
+			testName: "GMSA cred spec contents that are too long",
+			windowsOptions: &core.WindowsSecurityContextOptions{
+				GMSACredentialSpec: toPtr(strings.Repeat("a", maxGMSACredentialSpecLength+1)),
+			},
+			expectedErrorSubstring: "gmsaCredentialSpec size must be under",
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run("validateWindowsSecurityContextOptions with"+testCase.testName, func(t *testing.T) {
+			errs := validateWindowsSecurityContextOptions(testCase.windowsOptions, field.NewPath("field"))
+
+			switch len(errs) {
+			case 0:
+				if testCase.expectedErrorSubstring != "" {
+					t.Errorf("expected a failure containing the substring: %q", testCase.expectedErrorSubstring)
+				}
+			case 1:
+				if testCase.expectedErrorSubstring == "" {
+					t.Errorf("didn't expect a failure, got: %q", errs[0].Error())
+				} else if !strings.Contains(errs[0].Error(), testCase.expectedErrorSubstring) {
+					t.Errorf("expected a failure with the substring %q, got %q instead", testCase.expectedErrorSubstring, errs[0].Error())
+				}
+			default:
+				t.Errorf("got %d failures", len(errs))
+				for i, err := range errs {
+					t.Errorf("error %d: %q", i, err.Error())
+				}
+			}
+		})
+	}
+}

pkg/apis/core/zz_generated.deepcopy.go

@@ -3464,7 +3464,7 @@ func (in *PodSecurityContext) DeepCopyInto(out *PodSecurityContext) {
 	if in.WindowsOptions != nil {
 		in, out := &in.WindowsOptions, &out.WindowsOptions
 		*out = new(WindowsSecurityContextOptions)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.RunAsUser != nil {
 		in, out := &in.RunAsUser, &out.RunAsUser
@@ -4646,7 +4646,7 @@ func (in *SecurityContext) DeepCopyInto(out *SecurityContext) {
 	if in.WindowsOptions != nil {
 		in, out := &in.WindowsOptions, &out.WindowsOptions
 		*out = new(WindowsSecurityContextOptions)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.RunAsUser != nil {
 		in, out := &in.RunAsUser, &out.RunAsUser
@@ -5475,6 +5475,16 @@ func (in *WeightedPodAffinityTerm) DeepCopy() *WeightedPodAffinityTerm {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WindowsSecurityContextOptions) DeepCopyInto(out *WindowsSecurityContextOptions) {
 	*out = *in
+	if in.GMSACredentialSpecName != nil {
+		in, out := &in.GMSACredentialSpecName, &out.GMSACredentialSpecName
+		*out = new(string)
+		**out = **in
+	}
+	if in.GMSACredentialSpec != nil {
+		in, out := &in.GMSACredentialSpec, &out.GMSACredentialSpec
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 

pkg/kubelet/dockershim/docker_container_windows.go

@@ -30,7 +30,6 @@ import (
 	dockercontainer "github.com/docker/docker/api/types/container"
 
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
-	"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
 )
 
 type containerCleanupInfo struct {
@@ -54,7 +53,7 @@ func (ds *dockerService) applyPlatformSpecificDockerConfig(request *runtimeapi.C
 	return cleanupInfo, nil
 }
 
-// applyGMSAConfig looks at the kuberuntime.GMSASpecContainerAnnotationKey container annotation; if present,
+// applyGMSAConfig looks at the container's .Windows.SecurityContext.GMSACredentialSpec field; if present,
 // it copies its contents to a unique registry value, and sets a SecurityOpt on the config pointing to that registry value.
 // We use registry values instead of files since their location cannot change - as opposed to credential spec files,
 // whose location could potentially change down the line, or even be unknown (eg if docker is not installed on the
@@ -63,7 +62,10 @@ func (ds *dockerService) applyPlatformSpecificDockerConfig(request *runtimeapi.C
 // as it will avoid cluttering the registry - there is a moby PR out for this:
 // https://github.com/moby/moby/pull/38777
 func applyGMSAConfig(config *runtimeapi.ContainerConfig, createConfig *dockertypes.ContainerCreateConfig, cleanupInfo *containerCleanupInfo) error {
-	credSpec := config.Annotations[kuberuntime.GMSASpecContainerAnnotationKey]
+	var credSpec string
+	if config.Windows != nil && config.Windows.SecurityContext != nil {
+		credSpec = config.Windows.SecurityContext.CredentialSpec
+	}
 	if credSpec == "" {
 		return nil
 	}

pkg/kubelet/dockershim/docker_container_windows_test.go

@@ -73,7 +73,11 @@ func TestApplyGMSAConfig(t *testing.T) {
 	expectedValueName := "k8s-cred-spec-" + expectedHex
 
 	containerConfigWithGMSAAnnotation := &runtimeapi.ContainerConfig{
-		Annotations: map[string]string{"container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec},
+		Windows: &runtimeapi.WindowsContainerConfig{
+			SecurityContext: &runtimeapi.WindowsContainerSecurityContext{
+				CredentialSpec: dummyCredSpec,
+			},
+		},
 	}
 
 	t.Run("happy path", func(t *testing.T) {

pkg/kubelet/kuberuntime/BUILD

@@ -90,7 +90,6 @@ go_test(
         "instrumented_services_test.go",
         "kuberuntime_container_linux_test.go",
         "kuberuntime_container_test.go",
-        "kuberuntime_container_windows_test.go",
         "kuberuntime_gc_test.go",
         "kuberuntime_image_test.go",
         "kuberuntime_manager_test.go",

pkg/kubelet/kuberuntime/kuberuntime_container_windows.go

@@ -36,12 +36,8 @@ func (m *kubeGenericRuntimeManager) applyPlatformSpecificContainerConfig(config
 	if err != nil {
 		return err
 	}
-
-	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) {
-		determineEffectiveSecurityContext(config, container, pod)
-	}
-
 	config.Windows = windowsConfig
+
 	return nil
 }
 
@@ -100,43 +96,11 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1
 	if username != "" {
 		wc.SecurityContext.RunAsUsername = username
 	}
-
-	return wc, nil
-}
-
-const (
-	// GMSASpecContainerAnnotationKey is the container annotation where we store the contents of the GMSA credential spec to use.
-	GMSASpecContainerAnnotationKey = "container.alpha.windows.kubernetes.io/gmsa-credential-spec"
-	// gMSAContainerSpecPodAnnotationKeySuffix is the suffix of the pod annotation where the GMSA webhook admission controller
-	// stores the contents of the GMSA credential spec for a given container (the full annotation being the container's name
-	// with this suffix appended).
-	gMSAContainerSpecPodAnnotationKeySuffix = "." + GMSASpecContainerAnnotationKey
-	// gMSAPodSpecPodAnnotationKey is the pod annotation where the GMSA webhook admission controller stores the contents of the GMSA
-	// credential spec to use for containers that do not have their own specific GMSA cred spec set via a
-	// gMSAContainerSpecPodAnnotationKeySuffix annotation as explained above
-	gMSAPodSpecPodAnnotationKey = "pod.alpha.windows.kubernetes.io/gmsa-credential-spec"
-)
-
-// determineEffectiveSecurityContext determines the effective GMSA credential spec and, if any, copies it to the container's
-// GMSASpecContainerAnnotationKey annotation.
-func determineEffectiveSecurityContext(config *runtimeapi.ContainerConfig, container *v1.Container, pod *v1.Pod) {
-	var containerCredSpec string
-
-	containerGMSAPodAnnotation := container.Name + gMSAContainerSpecPodAnnotationKeySuffix
-	if pod.Annotations[containerGMSAPodAnnotation] != "" {
-		containerCredSpec = pod.Annotations[containerGMSAPodAnnotation]
-	} else if pod.Annotations[gMSAPodSpecPodAnnotationKey] != "" {
-		containerCredSpec = pod.Annotations[gMSAPodSpecPodAnnotationKey]
+	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) &&
+		effectiveSc.WindowsOptions != nil &&
+		effectiveSc.WindowsOptions.GMSACredentialSpec != nil {
+		wc.SecurityContext.CredentialSpec = *effectiveSc.WindowsOptions.GMSACredentialSpec
 	}
 
-	if containerCredSpec != "" {
-		if config.Annotations == nil {
-			config.Annotations = make(map[string]string)
-		}
-		config.Annotations[GMSASpecContainerAnnotationKey] = containerCredSpec
-	} else {
-		// the annotation shouldn't be present, but let's err on the side of caution:
-		// it should only be set here and nowhere else
-		delete(config.Annotations, GMSASpecContainerAnnotationKey)
-	}
+	return wc, nil
 }

pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go

-/*
-Copyright 2019 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kuberuntime
-
-import (
-	"github.com/stretchr/testify/assert"
-	"testing"
-
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
-)
-
-func TestDetermineEffectiveSecurityContext(t *testing.T) {
-	containerName := "container_name"
-	container := &corev1.Container{Name: containerName}
-	dummyCredSpec := "test cred spec contents"
-
-	buildPod := func(annotations map[string]string) *corev1.Pod {
-		return &corev1.Pod{
-			ObjectMeta: metav1.ObjectMeta{
-				Annotations: annotations,
-			},
-		}
-	}
-
-	t.Run("when there's a specific GMSA for that container, and no pod-wide GMSA", func(t *testing.T) {
-		containerConfig := &runtimeapi.ContainerConfig{}
-
-		pod := buildPod(map[string]string{
-			"container_name.container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec,
-		})
-
-		determineEffectiveSecurityContext(containerConfig, container, pod)
-
-		assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"])
-	})
-	t.Run("when there's a specific GMSA for that container, and a pod-wide GMSA", func(t *testing.T) {
-		containerConfig := &runtimeapi.ContainerConfig{}
-
-		pod := buildPod(map[string]string{
-			"container_name.container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec,
-			"pod.alpha.windows.kubernetes.io/gmsa-credential-spec":                      "should be ignored",
-		})
-
-		determineEffectiveSecurityContext(containerConfig, container, pod)
-
-		assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"])
-	})
-	t.Run("when there's no specific GMSA for that container, and a pod-wide GMSA", func(t *testing.T) {
-		containerConfig := &runtimeapi.ContainerConfig{}
-
-		pod := buildPod(map[string]string{
-			"pod.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec,
-		})
-
-		determineEffectiveSecurityContext(containerConfig, container, pod)
-
-		assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"])
-	})
-	t.Run("when there's no specific GMSA for that container, and no pod-wide GMSA", func(t *testing.T) {
-		containerConfig := &runtimeapi.ContainerConfig{}
-
-		determineEffectiveSecurityContext(containerConfig, container, &corev1.Pod{})
-
-		assert.Nil(t, containerConfig.Annotations)
-	})
-}

pkg/securitycontext/util.go

@@ -66,6 +66,18 @@ func DetermineEffectiveSecurityContext(pod *v1.Pod, container *v1.Container) *v1
 		*effectiveSc.SELinuxOptions = *containerSc.SELinuxOptions
 	}
 
+	if containerSc.WindowsOptions != nil {
+		// only override fields that are set at the container level, not the whole thing
+		if effectiveSc.WindowsOptions == nil {
+			effectiveSc.WindowsOptions = &v1.WindowsSecurityContextOptions{}
+		}
+		if containerSc.WindowsOptions.GMSACredentialSpecName != nil || containerSc.WindowsOptions.GMSACredentialSpec != nil {
+			// both GMSA fields go hand in hand
+			effectiveSc.WindowsOptions.GMSACredentialSpecName = containerSc.WindowsOptions.GMSACredentialSpecName
+			effectiveSc.WindowsOptions.GMSACredentialSpec = containerSc.WindowsOptions.GMSACredentialSpec
+		}
+	}
+
 	if containerSc.Capabilities != nil {
 		effectiveSc.Capabilities = new(v1.Capabilities)
 		*effectiveSc.Capabilities = *containerSc.Capabilities
@@ -120,6 +132,12 @@ func securityContextFromPodSecurityContext(pod *v1.Pod) *v1.SecurityContext {
 		synthesized.SELinuxOptions = &v1.SELinuxOptions{}
 		*synthesized.SELinuxOptions = *pod.Spec.SecurityContext.SELinuxOptions
 	}
+
+	if pod.Spec.SecurityContext.WindowsOptions != nil {
+		synthesized.WindowsOptions = &v1.WindowsSecurityContextOptions{}
+		*synthesized.WindowsOptions = *pod.Spec.SecurityContext.WindowsOptions
+	}
+
 	if pod.Spec.SecurityContext.RunAsUser != nil {
 		synthesized.RunAsUser = new(int64)
 		*synthesized.RunAsUser = *pod.Spec.SecurityContext.RunAsUser

staging/src/k8s.io/api/core/v1/generated.proto

@@ -4863,5 +4863,16 @@ message WeightedPodAffinityTerm {
 
 // WindowsSecurityContextOptions contain Windows-specific options and credentials.
 message WindowsSecurityContextOptions {
+  // GMSACredentialSpecName is the name of the GMSA credential spec to use.
+  // This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.
+  // +optional
+  optional string gmsaCredentialSpecName = 1;
+
+  // GMSACredentialSpec is where the GMSA admission webhook
+  // (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the
+  // GMSA credential spec named by the GMSACredentialSpecName field.
+  // This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.
+  // +optional
+  optional string gmsaCredentialSpec = 2;
 }
 

staging/src/k8s.io/api/core/v1/types.go

@@ -5355,7 +5355,17 @@ type SELinuxOptions struct {
 
 // WindowsSecurityContextOptions contain Windows-specific options and credentials.
 type WindowsSecurityContextOptions struct {
-	// intentionally left empty for now
+	// GMSACredentialSpecName is the name of the GMSA credential spec to use.
+	// This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.
+	// +optional
+	GMSACredentialSpecName *string `json:"gmsaCredentialSpecName,omitempty" protobuf:"bytes,1,opt,name=gmsaCredentialSpecName"`
+
+	// GMSACredentialSpec is where the GMSA admission webhook
+	// (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the
+	// GMSA credential spec named by the GMSACredentialSpecName field.
+	// This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.
+	// +optional
+	GMSACredentialSpec *string `json:"gmsaCredentialSpec,omitempty" protobuf:"bytes,2,opt,name=gmsaCredentialSpec"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go

@@ -2363,7 +2363,9 @@ func (WeightedPodAffinityTerm) SwaggerDoc() map[string]string {
 }
 
 var map_WindowsSecurityContextOptions = map[string]string{
-	"": "WindowsSecurityContextOptions contain Windows-specific options and credentials.",
+	"":                       "WindowsSecurityContextOptions contain Windows-specific options and credentials.",
+	"gmsaCredentialSpecName": "GMSACredentialSpecName is the name of the GMSA credential spec to use. This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.",
+	"gmsaCredentialSpec":     "GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.",
 }
 
 func (WindowsSecurityContextOptions) SwaggerDoc() map[string]string {

staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go

@@ -3457,7 +3457,7 @@ func (in *PodSecurityContext) DeepCopyInto(out *PodSecurityContext) {
 	if in.WindowsOptions != nil {
 		in, out := &in.WindowsOptions, &out.WindowsOptions
 		*out = new(WindowsSecurityContextOptions)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.RunAsUser != nil {
 		in, out := &in.RunAsUser, &out.RunAsUser
@@ -4656,7 +4656,7 @@ func (in *SecurityContext) DeepCopyInto(out *SecurityContext) {
 	if in.WindowsOptions != nil {
 		in, out := &in.WindowsOptions, &out.WindowsOptions
 		*out = new(WindowsSecurityContextOptions)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.RunAsUser != nil {
 		in, out := &in.RunAsUser, &out.RunAsUser
@@ -5490,6 +5490,16 @@ func (in *WeightedPodAffinityTerm) DeepCopy() *WeightedPodAffinityTerm {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WindowsSecurityContextOptions) DeepCopyInto(out *WindowsSecurityContextOptions) {
 	*out = *in
+	if in.GMSACredentialSpecName != nil {
+		in, out := &in.GMSACredentialSpecName, &out.GMSACredentialSpecName
+		*out = new(string)
+		**out = **in
+	}
+	if in.GMSACredentialSpec != nil {
+		in, out := &in.GMSACredentialSpec, &out.GMSACredentialSpec
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 

staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto

@@ -637,6 +637,9 @@ message WindowsContainerSecurityContext {
     // exist in the container image and be resolved there by the runtime;
     // otherwise, the runtime MUST return error.
     string run_as_username = 1;
+
+    // The contents of the GMSA credential spec to use to run this container.
+    string credential_spec = 2;
 }
 
 // WindowsContainerConfig contains platform-specific configuration for

test/e2e/windows/gmsa.go

@@ -46,24 +46,31 @@ var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA [Slow]", func(
 				container2Name := "container2"
 				container2Domain := "contoso.org"
 
-				containers := make([]corev1.Container, 2)
-				for i, name := range []string{container1Name, container2Name} {
-					containers[i] = corev1.Container{
-						Name:  name,
-						Image: imageutils.GetPauseImageName(),
-					}
-				}
-
 				pod := &corev1.Pod{
 					ObjectMeta: metav1.ObjectMeta{
 						Name: podName,
-						Annotations: map[string]string{
-							"pod.alpha.windows.kubernetes.io/gmsa-credential-spec":                         generateDummyCredSpecs(podDomain),
-							container2Name + ".container.alpha.windows.kubernetes.io/gmsa-credential-spec": generateDummyCredSpecs(container2Domain),
-						},
 					},
 					Spec: corev1.PodSpec{
-						Containers: containers,
+						Containers: []corev1.Container{
+							{
+								Name:  container1Name,
+								Image: imageutils.GetPauseImageName(),
+							},
+							{
+								Name:  container2Name,
+								Image: imageutils.GetPauseImageName(),
+								SecurityContext: &corev1.SecurityContext{
+									WindowsOptions: &corev1.WindowsSecurityContextOptions{
+										GMSACredentialSpec: generateDummyCredSpecs(container2Domain),
+									},
+								},
+							},
+						},
+						SecurityContext: &corev1.PodSecurityContext{
+							WindowsOptions: &corev1.WindowsSecurityContextOptions{
+								GMSACredentialSpec: generateDummyCredSpecs(podDomain),
+							},
+						},
 					},
 				}
 
@@ -108,10 +115,10 @@ var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA [Slow]", func(
 	})
 })
 
-func generateDummyCredSpecs(domain string) string {
+func generateDummyCredSpecs(domain string) *string {
 	shortName := strings.ToUpper(strings.Split(domain, ".")[0])
 
-	return fmt.Sprintf(`{
+	credSpecs := fmt.Sprintf(`{
        "ActiveDirectoryConfig":{
           "GroupManagedServiceAccounts":[
              {
@@ -136,4 +143,6 @@ func generateDummyCredSpecs(domain string) string {
           "Sid":"S-1-5-21-2126729477-2524175714-3194792973"
        }
     }`, shortName, domain, domain, domain, shortName)
+
+	return &credSpecs
 }