Feature-Gate affinity in annotations

pkg/api/v1/helpers.go

@@ -27,7 +27,9 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/features"
 )
 
 // IsOpaqueIntResourceName returns true if the resource name has the opaque
@@ -277,6 +279,10 @@ const (
 	// an object (e.g. secret, config map) before fetching it again from apiserver.
 	// This annotation can be attached to node.
 	ObjectTTLAnnotationKey string = "node.alpha.kubernetes.io/ttl"
+
+	// AffinityAnnotationKey represents the key of affinity data (json serialized)
+	// in the Annotations of a Pod.  TODO remove in 1.7
+	AffinityAnnotationKey string = "scheduler.alpha.kubernetes.io/affinity"
 )
 
 // GetTolerationsFromPodAnnotations gets the json serialized tolerations data from Pod.Annotations
@@ -597,3 +603,41 @@ func RemoveTaint(node *Node, taint *Taint) (*Node, bool, error) {
 	}
 	return newNode, true, nil
 }
+
+// GetAffinityFromPodAnnotations gets the json serialized affinity data from Pod.Annotations
+// and converts it to the Affinity type in api.
+// TODO remove for 1.7
+func GetAffinityFromPodAnnotations(annotations map[string]string) (*Affinity, error) {
+	if len(annotations) > 0 && annotations[AffinityAnnotationKey] != "" {
+		var affinity Affinity
+		err := json.Unmarshal([]byte(annotations[AffinityAnnotationKey]), &affinity)
+		if err != nil {
+			return nil, err
+		}
+		return &affinity, nil
+	}
+	return nil, nil
+}
+
+// Reconcile api and annotation affinity definitions.
+// TODO remove for 1.7
+func ReconcileAffinity(pod *Pod) *Affinity {
+	affinity := pod.Spec.Affinity
+	if utilfeature.DefaultFeatureGate.Enabled(features.AffinityInAnnotations) {
+		annotationsAffinity, _ := GetAffinityFromPodAnnotations(pod.Annotations)
+		if affinity == nil && annotationsAffinity != nil {
+			affinity = annotationsAffinity
+		} else if annotationsAffinity != nil {
+			if affinity != nil && affinity.NodeAffinity == nil && annotationsAffinity.NodeAffinity != nil {
+				affinity.NodeAffinity = annotationsAffinity.NodeAffinity
+			}
+			if affinity != nil && affinity.PodAffinity == nil && annotationsAffinity.PodAffinity != nil {
+				affinity.PodAffinity = annotationsAffinity.PodAffinity
+			}
+			if affinity != nil && affinity.PodAntiAffinity == nil && annotationsAffinity.PodAntiAffinity != nil {
+				affinity.PodAntiAffinity = annotationsAffinity.PodAntiAffinity
+			}
+		}
+	}
+	return affinity
+}

pkg/api/v1/helpers_test.go

@@ -17,12 +17,14 @@ limitations under the License.
 package v1
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 )
 
 func TestAddToNodeAddresses(t *testing.T) {
@@ -644,3 +646,180 @@ func TestSysctlsFromPodAnnotation(t *testing.T) {
 		}
 	}
 }
+
+func TestGetAffinityFromPodAnnotations(t *testing.T) {
+	testCases := []struct {
+		pod       *Pod
+		expectErr bool
+	}{
+		{
+			pod:       &Pod{},
+			expectErr: false,
+		},
+		{
+			pod: &Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						AffinityAnnotationKey: `
+						{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
+							"nodeSelectorTerms": [{
+								"matchExpressions": [{
+									"key": "foo",
+									"operator": "In",
+									"values": ["value1", "value2"]
+								}]
+							}]
+						}}}`,
+					},
+				},
+			},
+			expectErr: false,
+		},
+		{
+			pod: &Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						AffinityAnnotationKey: `
+						{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
+							"nodeSelectorTerms": [{
+								"matchExpressions": [{
+									"key": "foo",
+						`,
+					},
+				},
+			},
+			expectErr: true,
+		},
+	}
+
+	for i, tc := range testCases {
+		_, err := GetAffinityFromPodAnnotations(tc.pod.Annotations)
+		if err == nil && tc.expectErr {
+			t.Errorf("[%v]expected error but got none.", i)
+		}
+		if err != nil && !tc.expectErr {
+			t.Errorf("[%v]did not expect error but got: %v", i, err)
+		}
+	}
+}
+
+func TestReconcileAffinity(t *testing.T) {
+	baseAffinity := &Affinity{
+		NodeAffinity: &NodeAffinity{
+			RequiredDuringSchedulingIgnoredDuringExecution: &NodeSelector{
+				NodeSelectorTerms: []NodeSelectorTerm{
+					{
+						MatchExpressions: []NodeSelectorRequirement{
+							{
+								Key:      "foo",
+								Operator: NodeSelectorOpIn,
+								Values:   []string{"bar", "value2"},
+							},
+						},
+					},
+				},
+			},
+		},
+		PodAffinity: &PodAffinity{
+			RequiredDuringSchedulingIgnoredDuringExecution: []PodAffinityTerm{
+				{
+					LabelSelector: &metav1.LabelSelector{
+						MatchExpressions: []metav1.LabelSelectorRequirement{
+							{
+								Key:      "security",
+								Operator: metav1.LabelSelectorOpDoesNotExist,
+								Values:   []string{"securityscan"},
+							},
+						},
+					},
+					TopologyKey: "topologyKey1",
+				},
+			},
+		},
+		PodAntiAffinity: &PodAntiAffinity{
+			RequiredDuringSchedulingIgnoredDuringExecution: []PodAffinityTerm{
+				{
+					LabelSelector: &metav1.LabelSelector{
+						MatchExpressions: []metav1.LabelSelectorRequirement{
+							{
+								Key:      "service",
+								Operator: metav1.LabelSelectorOpIn,
+								Values:   []string{"S1", "value2"},
+							},
+						},
+					},
+					TopologyKey: "topologyKey2",
+					Namespaces:  []string{"ns1"},
+				},
+			},
+		},
+	}
+
+	nodeAffinityAnnotation := map[string]string{
+		AffinityAnnotationKey: `
+		{"nodeAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [
+			{
+				"weight": 2,
+				"preference": {"matchExpressions": [
+					{
+						"key": "foo",
+						"operator": "In", "values": ["bar"]
+					}
+				]}
+			}
+		]}}`,
+	}
+
+	testCases := []struct {
+		pod                *Pod
+		expected           *Affinity
+		annotationsEnabled bool
+	}{
+		{
+			pod: &Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: nodeAffinityAnnotation,
+				},
+				Spec: PodSpec{
+					Affinity: baseAffinity,
+				},
+			},
+			expected:           baseAffinity,
+			annotationsEnabled: true,
+		},
+		{
+			pod: &Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: nodeAffinityAnnotation,
+				},
+			},
+			expected: &Affinity{
+				NodeAffinity: &NodeAffinity{
+					PreferredDuringSchedulingIgnoredDuringExecution: []PreferredSchedulingTerm{
+						{
+							Weight: 2,
+							Preference: NodeSelectorTerm{
+								MatchExpressions: []NodeSelectorRequirement{
+									{
+										Key:      "foo",
+										Operator: NodeSelectorOpIn,
+										Values:   []string{"bar"},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			annotationsEnabled: true,
+		},
+	}
+
+	for i, tc := range testCases {
+		utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("AffinityInAnnotations=%t", tc.annotationsEnabled))
+		affinity := ReconcileAffinity(tc.pod)
+		if affinity != tc.expected {
+			t.Errorf("[%v]did not get expected affinity. got: %v instead of %v", i, affinity, tc.expected)
+		}
+	}
+}

pkg/features/kube_features.go

@@ -85,6 +85,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	DynamicVolumeProvisioning:                   {Default: true, PreRelease: utilfeature.Alpha},
 	ExperimentalHostUserNamespaceDefaultingGate: {Default: false, PreRelease: utilfeature.Beta},
 	ExperimentalCriticalPodAnnotation:           {Default: false, PreRelease: utilfeature.Alpha},
+	AffinityInAnnotations:                       {Default: false, PreRelease: utilfeature.Alpha},
 
 	// inherited features from generic apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:

plugin/pkg/scheduler/algorithm/predicates/predicates.go

@@ -585,7 +585,7 @@ func podMatchesNodeLabels(pod *v1.Pod, node *v1.Node) bool {
 	// 5. zero-length non-nil []NodeSelectorRequirement matches no nodes also, just for simplicity
 	// 6. non-nil empty NodeSelectorRequirement is not allowed
 	nodeAffinityMatches := true
-	affinity := pod.Spec.Affinity
+	affinity := v1.ReconcileAffinity(pod)
 	if affinity != nil && affinity.NodeAffinity != nil {
 		nodeAffinity := affinity.NodeAffinity
 		// if no required NodeAffinity requirements, will do no-op, means select all nodes.
@@ -897,7 +897,7 @@ func (c *PodAffinityChecker) InterPodAffinityMatches(pod *v1.Pod, meta interface
 	}
 
 	// Now check if <pod> requirements will be satisfied on this node.
-	affinity := pod.Spec.Affinity
+	affinity := v1.ReconcileAffinity(pod)
 	if affinity == nil || (affinity.PodAffinity == nil && affinity.PodAntiAffinity == nil) {
 		return true, nil, nil
 	}
@@ -1001,7 +1001,7 @@ func getMatchingAntiAffinityTerms(pod *v1.Pod, nodeInfoMap map[string]*scheduler
 		}
 		var nodeResult []matchingPodAntiAffinityTerm
 		for _, existingPod := range nodeInfo.PodsWithAffinity() {
-			affinity := existingPod.Spec.Affinity
+			affinity := v1.ReconcileAffinity(existingPod)
 			if affinity == nil {
 				continue
 			}
@@ -1029,7 +1029,7 @@ func getMatchingAntiAffinityTerms(pod *v1.Pod, nodeInfoMap map[string]*scheduler
 func (c *PodAffinityChecker) getMatchingAntiAffinityTerms(pod *v1.Pod, allPods []*v1.Pod) ([]matchingPodAntiAffinityTerm, error) {
 	var result []matchingPodAntiAffinityTerm
 	for _, existingPod := range allPods {
-		affinity := existingPod.Spec.Affinity
+		affinity := v1.ReconcileAffinity(existingPod)
 		if affinity != nil && affinity.PodAntiAffinity != nil {
 			existingPodNode, err := c.info.GetNodeInfo(existingPod.Spec.NodeName)
 			if err != nil {

plugin/pkg/scheduler/algorithm/priorities/interpod_affinity.go

@@ -116,7 +116,7 @@ func (p *podAffinityPriorityMap) processTerms(terms []v1.WeightedPodAffinityTerm
 // Symmetry need to be considered for preferredDuringSchedulingIgnoredDuringExecution from podAffinity & podAntiAffinity,
 // symmetry need to be considered for hard requirements from podAffinity
 func (ipa *InterPodAffinity) CalculateInterPodAffinityPriority(pod *v1.Pod, nodeNameToInfo map[string]*schedulercache.NodeInfo, nodes []*v1.Node) (schedulerapi.HostPriorityList, error) {
-	affinity := pod.Spec.Affinity
+	affinity := v1.ReconcileAffinity(pod)
 	hasAffinityConstraints := affinity != nil && affinity.PodAffinity != nil
 	hasAntiAffinityConstraints := affinity != nil && affinity.PodAntiAffinity != nil
 
@@ -137,7 +137,7 @@ func (ipa *InterPodAffinity) CalculateInterPodAffinityPriority(pod *v1.Pod, node
 		if err != nil {
 			return err
 		}
-		existingPodAffinity := existingPod.Spec.Affinity
+		existingPodAffinity := v1.ReconcileAffinity(existingPod)
 		existingHasAffinityConstraints := existingPodAffinity != nil && existingPodAffinity.PodAffinity != nil
 		existingHasAntiAffinityConstraints := existingPodAffinity != nil && existingPodAffinity.PodAntiAffinity != nil
 

plugin/pkg/scheduler/algorithm/priorities/metadata.go

@@ -41,6 +41,6 @@ func PriorityMetadata(pod *v1.Pod, nodeNameToInfo map[string]*schedulercache.Nod
 	return &priorityMetadata{
 		nonZeroRequest: getNonZeroRequests(pod),
 		podTolerations: tolerations,
-		affinity:       pod.Spec.Affinity,
+		affinity:       v1.ReconcileAffinity(pod),
 	}
 }

plugin/pkg/scheduler/algorithm/priorities/node_affinity.go

@@ -42,7 +42,7 @@ func CalculateNodeAffinityPriorityMap(pod *v1.Pod, meta interface{}, nodeInfo *s
 		affinity = priorityMeta.affinity
 	} else {
 		// We couldn't parse metadata - fallback to the podspec.
-		affinity = pod.Spec.Affinity
+		affinity = v1.ReconcileAffinity(pod)
 	}
 
 	var count int32

plugin/pkg/scheduler/algorithm/priorities/node_affinity_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api/v1"
 	schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api"
 	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
@@ -177,3 +178,146 @@ func TestNodeAffinityPriority(t *testing.T) {
 		}
 	}
 }
+
+func TestNodeAffinityAnnotationsPriority(t *testing.T) {
+	utilfeature.DefaultFeatureGate.Set("AffinityInAnnotations=true")
+	label1 := map[string]string{"foo": "bar"}
+	label2 := map[string]string{"key": "value"}
+	label3 := map[string]string{"az": "az1"}
+	label4 := map[string]string{"abc": "az11", "def": "az22"}
+	label5 := map[string]string{"foo": "bar", "key": "value", "az": "az1"}
+
+	affinity1 := map[string]string{
+		v1.AffinityAnnotationKey: `
+		{"nodeAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [
+			{
+				"weight": 2,
+				"preference": {
+					"matchExpressions": [
+						{
+							"key": "foo",
+							"operator": "In", "values": ["bar"]
+						}
+					]
+				}
+			}
+		]}}`,
+	}
+
+	affinity2 := map[string]string{
+		v1.AffinityAnnotationKey: `
+		{"nodeAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [
+			{
+				"weight": 2,
+				"preference": {"matchExpressions": [
+					{
+						"key": "foo",
+						"operator": "In", "values": ["bar"]
+					}
+				]}
+			},
+			{
+				"weight": 4,
+				"preference": {"matchExpressions": [
+					{
+						"key": "key",
+						"operator": "In", "values": ["value"]
+					}
+				]}
+			},
+			{
+				"weight": 5,
+				"preference": {"matchExpressions": [
+					{
+						"key": "foo",
+						"operator": "In", "values": ["bar"]
+					},
+					{
+						"key": "key",
+						"operator": "In", "values": ["value"]
+					},
+					{
+						"key": "az",
+						"operator": "In", "values": ["az1"]
+					}
+				]}
+			}
+		]}}`,
+	}
+
+	tests := []struct {
+		pod          *v1.Pod
+		nodes        []*v1.Node
+		expectedList schedulerapi.HostPriorityList
+		test         string
+	}{
+		{
+			pod: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{},
+				},
+			},
+			nodes: []*v1.Node{
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label1}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label3}},
+			},
+			expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 0}, {Host: "machine2", Score: 0}, {Host: "machine3", Score: 0}},
+			test:         "all machines are same priority as NodeAffinity is nil",
+		},
+		{
+			pod: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: affinity1,
+				},
+			},
+			nodes: []*v1.Node{
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label4}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label3}},
+			},
+			expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 0}, {Host: "machine2", Score: 0}, {Host: "machine3", Score: 0}},
+			test:         "no machine macthes preferred scheduling requirements in NodeAffinity of pod so all machines' priority is zero",
+		},
+		{
+			pod: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: affinity1,
+				},
+			},
+			nodes: []*v1.Node{
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label1}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: label3}},
+			},
+			expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 10}, {Host: "machine2", Score: 0}, {Host: "machine3", Score: 0}},
+			test:         "only machine1 matches the preferred scheduling requirements of pod",
+		},
+		{
+			pod: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: affinity2,
+				},
+			},
+			nodes: []*v1.Node{
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: label1}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine5", Labels: label5}},
+				{ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: label2}},
+			},
+			expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 1}, {Host: "machine5", Score: 10}, {Host: "machine2", Score: 3}},
+			test:         "all machines matches the preferred scheduling requirements of pod but with different priorities ",
+		},
+	}
+
+	for _, test := range tests {
+		nodeNameToInfo := schedulercache.CreateNodeNameToInfoMap(nil, test.nodes)
+		nap := priorityFunction(CalculateNodeAffinityPriorityMap, CalculateNodeAffinityPriorityReduce)
+		list, err := nap(test.pod, nodeNameToInfo, test.nodes)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		if !reflect.DeepEqual(test.expectedList, list) {
+			t.Errorf("%s: \nexpected %#v, \ngot      %#v", test.test, test.expectedList, list)
+		}
+	}
+}

plugin/pkg/scheduler/schedulercache/node_info.go

@@ -217,7 +217,7 @@ func (n *NodeInfo) String() string {
 }
 
 func hasPodAffinityConstraints(pod *v1.Pod) bool {
-	affinity := pod.Spec.Affinity
+	affinity := v1.ReconcileAffinity(pod)
 	return affinity != nil && (affinity.PodAffinity != nil || affinity.PodAntiAffinity != nil)
 }
 

staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go

@@ -107,6 +107,8 @@ type FeatureGate interface {
 	// owner: @pweil-
 	// alpha: v1.5
 	ExperimentalHostUserNamespaceDefaulting() bool
+
+	AnninityInAnnotations() bool
 }
 
 // featureGate implements FeatureGate as well as pflag.Value for flag parsing.