Use PreferAvoidPods annotation to avoid pods being scheduled to specific node.

pkg/api/helpers.go

@@ -428,6 +428,10 @@ const (
 	// CreatedByAnnotation represents the key used to store the spec(json)
 	// used to create the resource.
 	CreatedByAnnotation = "kubernetes.io/created-by"
+
+	// PreferAvoidPodsAnnotationKey represents the key of preferAvoidPods data (json serialized)
+	// in the Annotations of a Node.
+	PreferAvoidPodsAnnotationKey string = "scheduler.alpha.kubernetes.io/preferAvoidPods"
 )
 
 // GetAffinityFromPod gets the json serialized affinity data from Pod.Annotations
@@ -500,3 +504,14 @@ func TaintToleratedByTolerations(taint *Taint, tolerations []Toleration) bool {
 	}
 	return tolerated
 }
+
+func GetAvoidPodsFromNodeAnnotations(annotations map[string]string) (AvoidPods, error) {
+	var avoidPods AvoidPods
+	if len(annotations) > 0 && annotations[PreferAvoidPodsAnnotationKey] != "" {
+		err := json.Unmarshal([]byte(annotations[PreferAvoidPodsAnnotationKey]), &avoidPods)
+		if err != nil {
+			return avoidPods, err
+		}
+	}
+	return avoidPods, nil
+}

pkg/api/helpers_test.go

@@ -295,3 +295,99 @@ func TestGetAffinityFromPod(t *testing.T) {
 		}
 	}
 }
+
+func TestGetAvoidPodsFromNode(t *testing.T) {
+	controllerFlag := true
+	testCases := []struct {
+		node        *Node
+		expectValue AvoidPods
+		expectErr   bool
+	}{
+		{
+			node:        &Node{},
+			expectValue: AvoidPods{},
+			expectErr:   false,
+		},
+		{
+			node: &Node{
+				ObjectMeta: ObjectMeta{
+					Annotations: map[string]string{
+						PreferAvoidPodsAnnotationKey: `
+							{
+							    "preferAvoidPods": [
+							        {
+							            "podSignature": {
+							                "podController": {
+						                            "apiVersion": "v1",
+						                            "kind": "ReplicationController",
+						                            "name": "foo",
+						                            "uid": "abcdef123456",
+						                            "controller": true
+							                }
+							            },
+							            "reason": "some reason",
+							            "message": "some message"
+							        }
+							    ]
+							}`,
+					},
+				},
+			},
+			expectValue: AvoidPods{
+				PreferAvoidPods: []PreferAvoidPodsEntry{
+					{
+						PodSignature: PodSignature{
+							PodController: &OwnerReference{
+								APIVersion: "v1",
+								Kind:       "ReplicationController",
+								Name:       "foo",
+								UID:        "abcdef123456",
+								Controller: &controllerFlag,
+							},
+						},
+						Reason:  "some reason",
+						Message: "some message",
+					},
+				},
+			},
+			expectErr: false,
+		},
+		{
+			node: &Node{
+				// Missing end symbol of "podController" and "podSignature"
+				ObjectMeta: ObjectMeta{
+					Annotations: map[string]string{
+						PreferAvoidPodsAnnotationKey: `
+							{
+							    "preferAvoidPods": [
+							        {
+							            "podSignature": {
+							                "podController": {
+							                    "kind": "ReplicationController",
+							                    "apiVersion": "v1"
+							            "reason": "some reason",
+							            "message": "some message"
+							        }
+							    ]
+							}`,
+					},
+				},
+			},
+			expectValue: AvoidPods{},
+			expectErr:   true,
+		},
+	}
+
+	for i, tc := range testCases {
+		v, err := GetAvoidPodsFromNodeAnnotations(tc.node.Annotations)
+		if err == nil && tc.expectErr {
+			t.Errorf("[%v]expected error but got none.", i)
+		}
+		if err != nil && !tc.expectErr {
+			t.Errorf("[%v]did not expect error but got: %v", i, err)
+		}
+		if !reflect.DeepEqual(tc.expectValue, v) {
+			t.Errorf("[%v]expect value %v but got %v with %v", i, tc.expectValue, v, v.PreferAvoidPods[0].PodSignature.PodController.Controller)
+		}
+	}
+}

pkg/api/types.go

@@ -2008,6 +2008,34 @@ type AttachedVolume struct {
 	DevicePath string `json:"devicePath"`
 }
 
+// AvoidPods describes pods that should avoid this node. This is the value for a
+// Node annotation with key scheduler.alpha.kubernetes.io/preferAvoidPods and
+// will eventually become a field of NodeStatus.
+type AvoidPods struct {
+	// Bounded-sized list of signatures of pods that should avoid this node, sorted
+	// in timestamp order from oldest to newest. Size of the slice is unspecified.
+	PreferAvoidPods []PreferAvoidPodsEntry `json:"preferAvoidPods,omitempty"`
+}
+
+// Describes a class of pods that should avoid this node.
+type PreferAvoidPodsEntry struct {
+	// The class of pods.
+	PodSignature PodSignature `json:"podSignature"`
+	// Time at which this entry was added to the list.
+	EvictionTime unversioned.Time `json:"evictionTime,omitempty"`
+	// (brief) reason why this entry was added to the list.
+	Reason string `json:"reason,omitempty"`
+	// Human readable message indicating why this entry was added to the list.
+	Message string `json:"message,omitempty"`
+}
+
+// Describes the class of pods that should avoid this node.
+// Exactly one field should be set.
+type PodSignature struct {
+	// Reference to controller whose pods should avoid this node.
+	PodController *OwnerReference `json:"podController,omitempty"`
+}
+
 // Describe a container image
 type ContainerImage struct {
 	// Names by which this image is known.

pkg/api/v1/generated.proto

@@ -80,6 +80,15 @@ message AttachedVolume {
   optional string devicePath = 2;
 }
 
+// AvoidPods describes pods that should avoid this node. This is the value for a
+// Node annotation with key scheduler.alpha.kubernetes.io/preferAvoidPods and
+// will eventually become a field of NodeStatus.
+message AvoidPods {
+  // Bounded-sized list of signatures of pods that should avoid this node, sorted
+  // in timestamp order from oldest to newest. Size of the slice is unspecified.
+  repeated PreferAvoidPodsEntry preferAvoidPods = 1;
+}
+
 // AzureFile represents an Azure File Service mount on the host and bind mount to the pod.
 message AzureFileVolumeSource {
   // the name of secret that contains Azure Storage Account Name and Key
@@ -2040,6 +2049,13 @@ message PodSecurityContext {
   optional int64 fsGroup = 5;
 }
 
+// Describes the class of pods that should avoid this node.
+// Exactly one field should be set.
+message PodSignature {
+  // Reference to controller whose pods should avoid this node.
+  optional OwnerReference podController = 1;
+}
+
 // PodSpec is a description of a pod.
 message PodSpec {
   // List of volumes that can be mounted by containers belonging to the pod.
@@ -2219,6 +2235,21 @@ message Preconditions {
   optional string uid = 1;
 }
 
+// Describes a class of pods that should avoid this node.
+message PreferAvoidPodsEntry {
+  // The class of pods.
+  optional PodSignature podSignature = 1;
+
+  // Time at which this entry was added to the list.
+  optional k8s.io.kubernetes.pkg.api.unversioned.Time evictionTime = 2;
+
+  // (brief) reason why this entry was added to the list.
+  optional string reason = 3;
+
+  // Human readable message indicating why this entry was added to the list.
+  optional string message = 4;
+}
+
 // An empty preferred scheduling term matches all objects with implicit weight 0
 // (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).
 message PreferredSchedulingTerm {

pkg/api/v1/types.go

@@ -2408,6 +2408,34 @@ type AttachedVolume struct {
 	DevicePath string `json:"devicePath" protobuf:"bytes,2,rep,name=devicePath"`
 }
 
+// AvoidPods describes pods that should avoid this node. This is the value for a
+// Node annotation with key scheduler.alpha.kubernetes.io/preferAvoidPods and
+// will eventually become a field of NodeStatus.
+type AvoidPods struct {
+	// Bounded-sized list of signatures of pods that should avoid this node, sorted
+	// in timestamp order from oldest to newest. Size of the slice is unspecified.
+	PreferAvoidPods []PreferAvoidPodsEntry `json:"preferAvoidPods,omitempty" protobuf:"bytes,1,rep,name=preferAvoidPods"`
+}
+
+// Describes a class of pods that should avoid this node.
+type PreferAvoidPodsEntry struct {
+	// The class of pods.
+	PodSignature PodSignature `json:"podSignature" protobuf:"bytes,1,opt,name=podSignature"`
+	// Time at which this entry was added to the list.
+	EvictionTime unversioned.Time `json:"evictionTime,omitempty" protobuf:"bytes,2,opt,name=evictionTime"`
+	// (brief) reason why this entry was added to the list.
+	Reason string `json:"reason,omitempty" protobuf:"bytes,3,opt,name=reason"`
+	// Human readable message indicating why this entry was added to the list.
+	Message string `json:"message,omitempty" protobuf:"bytes,4,opt,name=message"`
+}
+
+// Describes the class of pods that should avoid this node.
+// Exactly one field should be set.
+type PodSignature struct {
+	// Reference to controller whose pods should avoid this node.
+	PodController *OwnerReference `json:"podController,omitempty" protobuf:"bytes,1,opt,name=podController"`
+}
+
 // Describe a container image
 type ContainerImage struct {
 	// Names by which this image is known.

pkg/api/v1/types_swagger_doc_generated.go

@@ -60,6 +60,15 @@ func (AttachedVolume) SwaggerDoc() map[string]string {
 	return map_AttachedVolume
 }
 
+var map_AvoidPods = map[string]string{
+	"":                "AvoidPods describes pods that should avoid this node. This is the value for a Node annotation with key scheduler.alpha.kubernetes.io/preferAvoidPods and will eventually become a field of NodeStatus.",
+	"preferAvoidPods": "Bounded-sized list of signatures of pods that should avoid this node, sorted in timestamp order from oldest to newest. Size of the slice is unspecified.",
+}
+
+func (AvoidPods) SwaggerDoc() map[string]string {
+	return map_AvoidPods
+}
+
 var map_AzureFileVolumeSource = map[string]string{
 	"":           "AzureFile represents an Azure File Service mount on the host and bind mount to the pod.",
 	"secretName": "the name of secret that contains Azure Storage Account Name and Key",
@@ -1227,6 +1236,15 @@ func (PodSecurityContext) SwaggerDoc() map[string]string {
 	return map_PodSecurityContext
 }
 
+var map_PodSignature = map[string]string{
+	"":              "Describes the class of pods that should avoid this node. Exactly one field should be set.",
+	"podController": "Reference to controller whose pods should avoid this node.",
+}
+
+func (PodSignature) SwaggerDoc() map[string]string {
+	return map_PodSignature
+}
+
 var map_PodSpec = map[string]string{
 	"":                              "PodSpec is a description of a pod.",
 	"volumes":                       "List of volumes that can be mounted by containers belonging to the pod. More info: http://releases.k8s.io/HEAD/docs/user-guide/volumes.md",
@@ -1317,6 +1335,18 @@ func (Preconditions) SwaggerDoc() map[string]string {
 	return map_Preconditions
 }
 
+var map_PreferAvoidPodsEntry = map[string]string{
+	"":             "Describes a class of pods that should avoid this node.",
+	"podSignature": "The class of pods.",
+	"evictionTime": "Time at which this entry was added to the list.",
+	"reason":       "(brief) reason why this entry was added to the list.",
+	"message":      "Human readable message indicating why this entry was added to the list.",
+}
+
+func (PreferAvoidPodsEntry) SwaggerDoc() map[string]string {
+	return map_PreferAvoidPodsEntry
+}
+
 var map_PreferredSchedulingTerm = map[string]string{
 	"":           "An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).",
 	"weight":     "Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.",

pkg/api/validation/validation.go

@@ -1724,6 +1724,41 @@ func ValidateNodeSelector(nodeSelector *api.NodeSelector, fldPath *field.Path) f
 	return allErrs
 }
 
+// ValidateAvoidPodsInNodeAnnotations tests that the serialized AvoidPods in Node.Annotations has valid data
+func ValidateAvoidPodsInNodeAnnotations(annotations map[string]string, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	avoids, err := api.GetAvoidPodsFromNodeAnnotations(annotations)
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("AvoidPods"), api.PreferAvoidPodsAnnotationKey, err.Error()))
+		return allErrs
+	}
+
+	if len(avoids.PreferAvoidPods) != 0 {
+		for i, pa := range avoids.PreferAvoidPods {
+			idxPath := fldPath.Child(api.PreferAvoidPodsAnnotationKey).Index(i)
+			allErrs = append(allErrs, validatePreferAvoidPodsEntry(pa, idxPath)...)
+		}
+	}
+
+	return allErrs
+}
+
+// validatePreferAvoidPodsEntry tests if given PreferAvoidPodsEntry has valid data.
+func validatePreferAvoidPodsEntry(avoidPodEntry api.PreferAvoidPodsEntry, fldPath *field.Path) field.ErrorList {
+	allErrors := field.ErrorList{}
+	if avoidPodEntry.PodSignature.PodController == nil {
+		allErrors = append(allErrors, field.Required(fldPath.Child("PodSignature"), ""))
+	} else {
+		if *(avoidPodEntry.PodSignature.PodController.Controller) != true {
+			allErrors = append(allErrors,
+				field.Invalid(fldPath.Child("PodSignature").Child("PodController").Child("Controller"),
+					*(avoidPodEntry.PodSignature.PodController.Controller), "must point to a controller"))
+		}
+	}
+	return allErrors
+}
+
 // ValidatePreferredSchedulingTerms tests that the specified SoftNodeAffinity fields has valid data
 func ValidatePreferredSchedulingTerms(terms []api.PreferredSchedulingTerm, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
@@ -2359,10 +2394,14 @@ func ValidateTaintsInNodeAnnotations(annotations map[string]string, fldPath *fie
 }
 
 func ValidateNodeSpecificAnnotations(annotations map[string]string, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if annotations[api.PreferAvoidPodsAnnotationKey] != "" {
+		allErrs = append(allErrs, ValidateAvoidPodsInNodeAnnotations(annotations, fldPath)...)
+	}
 	if annotations[api.TaintsAnnotationKey] != "" {
-		return ValidateTaintsInNodeAnnotations(annotations, fldPath)
+		allErrs = append(allErrs, ValidateTaintsInNodeAnnotations(annotations, fldPath)...)
 	}
-	return field.ErrorList{}
+	return allErrs
 }
 
 // ValidateNode tests if required fields in the node are set.

pkg/api/validation/validation_test.go

@@ -4375,6 +4375,43 @@ func TestValidateNode(t *testing.T) {
 				ExternalID: "external",
 			},
 		},
+		{
+			ObjectMeta: api.ObjectMeta{
+				Name: "abc",
+				Annotations: map[string]string{
+					api.PreferAvoidPodsAnnotationKey: `
+							{
+							    "preferAvoidPods": [
+							        {
+							            "podSignature": {
+							                "podController": {
+							                    "apiVersion": "v1",
+							                    "kind": "ReplicationController",
+							                    "name": "foo",
+							                    "uid": "abcdef123456",
+							                    "controller": true
+							                }
+							            },
+							            "reason": "some reason",
+							            "message": "some message"
+							        }
+							    ]
+							}`,
+				},
+			},
+			Status: api.NodeStatus{
+				Addresses: []api.NodeAddress{
+					{Type: api.NodeLegacyHostIP, Address: "something"},
+				},
+				Capacity: api.ResourceList{
+					api.ResourceName(api.ResourceCPU):    resource.MustParse("10"),
+					api.ResourceName(api.ResourceMemory): resource.MustParse("0"),
+				},
+			},
+			Spec: api.NodeSpec{
+				ExternalID: "external",
+			},
+		},
 	}
 	for _, successCase := range successCases {
 		if errs := ValidateNode(&successCase); len(errs) != 0 {
@@ -4539,6 +4576,67 @@ func TestValidateNode(t *testing.T) {
 				ExternalID: "external",
 			},
 		},
+		"missing-podSignature": {
+			ObjectMeta: api.ObjectMeta{
+				Name: "abc-123",
+				Annotations: map[string]string{
+					api.PreferAvoidPodsAnnotationKey: `
+							{
+							    "preferAvoidPods": [
+							        {
+							            "reason": "some reason",
+							            "message": "some message"
+							        }
+							    ]
+							}`,
+				},
+			},
+			Status: api.NodeStatus{
+				Addresses: []api.NodeAddress{},
+				Capacity: api.ResourceList{
+					api.ResourceName(api.ResourceCPU):    resource.MustParse("10"),
+					api.ResourceName(api.ResourceMemory): resource.MustParse("0"),
+				},
+			},
+			Spec: api.NodeSpec{
+				ExternalID: "external",
+			},
+		},
+		"invalid-podController": {
+			ObjectMeta: api.ObjectMeta{
+				Name: "abc-123",
+				Annotations: map[string]string{
+					api.PreferAvoidPodsAnnotationKey: `
+							{
+							    "preferAvoidPods": [
+							        {
+							            "podSignature": {
+							                "podController": {
+							                    "apiVersion": "v1",
+							                    "kind": "ReplicationController",
+							                    "name": "foo",
+                                                                           "uid": "abcdef123456",
+                                                                           "controller": false
+							                }
+							            },
+							            "reason": "some reason",
+							            "message": "some message"
+							        }
+							    ]
+							}`,
+				},
+			},
+			Status: api.NodeStatus{
+				Addresses: []api.NodeAddress{},
+				Capacity: api.ResourceList{
+					api.ResourceName(api.ResourceCPU):    resource.MustParse("10"),
+					api.ResourceName(api.ResourceMemory): resource.MustParse("0"),
+				},
+			},
+			Spec: api.NodeSpec{
+				ExternalID: "external",
+			},
+		},
 	}
 	for k, v := range errorCases {
 		errs := ValidateNode(&v)
@@ -4548,14 +4646,16 @@ func TestValidateNode(t *testing.T) {
 		for i := range errs {
 			field := errs[i].Field
 			expectedFields := map[string]bool{
-				"metadata.name":                                                       true,
-				"metadata.labels":                                                     true,
-				"metadata.annotations":                                                true,
-				"metadata.namespace":                                                  true,
-				"spec.externalID":                                                     true,
-				"metadata.annotations.scheduler.alpha.kubernetes.io/taints[0].key":    true,
-				"metadata.annotations.scheduler.alpha.kubernetes.io/taints[0].value":  true,
-				"metadata.annotations.scheduler.alpha.kubernetes.io/taints[0].effect": true,
+				"metadata.name":                                                                                               true,
+				"metadata.labels":                                                                                             true,
+				"metadata.annotations":                                                                                        true,
+				"metadata.namespace":                                                                                          true,
+				"spec.externalID":                                                                                             true,
+				"metadata.annotations.scheduler.alpha.kubernetes.io/taints[0].key":                                            true,
+				"metadata.annotations.scheduler.alpha.kubernetes.io/taints[0].value":                                          true,
+				"metadata.annotations.scheduler.alpha.kubernetes.io/taints[0].effect":                                         true,
+				"metadata.annotations.scheduler.alpha.kubernetes.io/preferAvoidPods[0].PodSignature":                          true,
+				"metadata.annotations.scheduler.alpha.kubernetes.io/preferAvoidPods[0].PodSignature.PodController.Controller": true,
 			}
 			if val, ok := expectedFields[field]; ok {
 				if !val {
@@ -4764,6 +4864,87 @@ func TestValidateNodeUpdate(t *testing.T) {
 				},
 			},
 		}, true},
+		{api.Node{
+			ObjectMeta: api.ObjectMeta{
+				Name: "foo",
+			},
+		}, api.Node{
+			ObjectMeta: api.ObjectMeta{
+				Name: "foo",
+				Annotations: map[string]string{
+					api.PreferAvoidPodsAnnotationKey: `
+							{
+							    "preferAvoidPods": [
+							        {
+							            "podSignature": {
+							                "podController": {
+							                    "apiVersion": "v1",
+							                    "kind": "ReplicationController",
+							                    "name": "foo",
+                                                                           "uid": "abcdef123456",
+                                                                           "controller": true
+							                }
+							            },
+							            "reason": "some reason",
+							            "message": "some message"
+							        }
+							    ]
+							}`,
+				},
+			},
+			Spec: api.NodeSpec{
+				Unschedulable: false,
+			},
+		}, true},
+		{api.Node{
+			ObjectMeta: api.ObjectMeta{
+				Name: "foo",
+			},
+		}, api.Node{
+			ObjectMeta: api.ObjectMeta{
+				Name: "foo",
+				Annotations: map[string]string{
+					api.PreferAvoidPodsAnnotationKey: `
+							{
+							    "preferAvoidPods": [
+							        {
+							            "reason": "some reason",
+							            "message": "some message"
+							        }
+							    ]
+							}`,
+				},
+			},
+		}, false},
+		{api.Node{
+			ObjectMeta: api.ObjectMeta{
+				Name: "foo",
+			},
+		}, api.Node{
+			ObjectMeta: api.ObjectMeta{
+				Name: "foo",
+				Annotations: map[string]string{
+					api.PreferAvoidPodsAnnotationKey: `
+							{
+							    "preferAvoidPods": [
+							        {
+							            "podSignature": {
+							                "podController": {
+							                    "apiVersion": "v1",
+							                    "kind": "ReplicationController",
+							                    "name": "foo",
+							                    "uid": "abcdef123456",
+							                    "controller": false
+							                }
+							            },
+							            "reason": "some reason",
+							            "message": "some message"
+							        }
+							    ]
+							}`,
+				},
+			},
+		}, false},
 	}
 	for i, test := range tests {
 		test.oldNode.ObjectMeta.ResourceVersion = "1"

plugin/pkg/scheduler/algorithm/priorities/priorities.go

@@ -277,3 +277,79 @@ func fractionOfCapacity(requested, capacity int64) float64 {
 	}
 	return float64(requested) / float64(capacity)
 }
+
+type NodePreferAvoidPod struct {
+	controllerLister algorithm.ControllerLister
+	replicaSetLister algorithm.ReplicaSetLister
+}
+
+func NewNodePreferAvoidPodsPriority(controllerLister algorithm.ControllerLister, replicaSetLister algorithm.ReplicaSetLister) algorithm.PriorityFunction {
+	nodePreferAvoid := &NodePreferAvoidPod{
+		controllerLister: controllerLister,
+		replicaSetLister: replicaSetLister,
+	}
+	return nodePreferAvoid.CalculateNodePreferAvoidPodsPriority
+}
+
+func (npa *NodePreferAvoidPod) CalculateNodePreferAvoidPodsPriority(pod *api.Pod, nodeNameToInfo map[string]*schedulercache.NodeInfo, nodeLister algorithm.NodeLister) (schedulerapi.HostPriorityList, error) {
+	var score int
+	nodes, err := nodeLister.List()
+	if err != nil {
+		return nil, err
+	}
+
+	result := []schedulerapi.HostPriority{}
+
+	// TODO: Once we have ownerReference fully implemented, use it to find controller for the pod.
+	rcs, err := npa.controllerLister.GetPodControllers(pod)
+	rss, err := npa.replicaSetLister.GetPodReplicaSets(pod)
+	if len(rcs) == 0 && len(rss) == 0 {
+		for _, node := range nodes {
+			result = append(result, schedulerapi.HostPriority{Host: node.Name, Score: 10})
+		}
+		return result, nil
+	}
+
+	avoidNodes := map[string]bool{}
+	for _, node := range nodes {
+		avoidNodes[node.Name] = false
+
+		avoids, err := api.GetAvoidPodsFromNodeAnnotations(node.Annotations)
+		if err != nil {
+			continue
+		}
+
+		for _, avoid := range avoids.PreferAvoidPods {
+			for _, rc := range rcs {
+				if avoid.PodSignature.PodController.Kind == "ReplicationController" && avoid.PodSignature.PodController.UID == rc.UID {
+					avoidNodes[node.Name] = true
+					break
+				}
+			}
+			if avoidNodes[node.Name] {
+				break
+			}
+			for _, rs := range rss {
+				if avoid.PodSignature.PodController.Kind == "ReplicaSet" && avoid.PodSignature.PodController.UID == rs.UID {
+					avoidNodes[node.Name] = true
+					break
+				}
+			}
+			if avoidNodes[node.Name] {
+				break
+			}
+		}
+	}
+
+	//score int - scale of 0-10
+	// 0 being the lowest priority and 10 being the highest
+	for nodeName, shouldAvoid := range avoidNodes {
+		if shouldAvoid {
+			score = 0
+		} else {
+			score = 10
+		}
+		result = append(result, schedulerapi.HostPriority{Host: nodeName, Score: score})
+	}
+	return result, nil
+}

plugin/pkg/scheduler/algorithm/priorities/priorities_test.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/kubernetes/cmd/libs/go2idl/types"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/resource"
+	"k8s.io/kubernetes/pkg/api/unversioned"
 	"k8s.io/kubernetes/pkg/apis/extensions"
 	"k8s.io/kubernetes/pkg/util/codeinspector"
 	"k8s.io/kubernetes/plugin/pkg/scheduler"
@@ -930,3 +931,137 @@ func TestPrioritiesRegistered(t *testing.T) {
 		}
 	}
 }
+
+func TestNodePreferAvoidPriority(t *testing.T) {
+	label1 := map[string]string{"foo": "bar"}
+	label2 := map[string]string{"bar": "foo"}
+	annotations1 := map[string]string{
+		api.PreferAvoidPodsAnnotationKey: `
+							{
+							    "preferAvoidPods": [
+							        {
+							            "podSignature": {
+							                "podController": {
+							                    "apiVersion": "v1",
+							                    "kind": "ReplicationController",
+							                    "name": "foo",
+							                    "uid": "abcdef123456",
+							                    "controller": true
+							                }
+							            },
+							            "reason": "some reason",
+							            "message": "some message"
+							        }
+							    ]
+							}`,
+	}
+	annotations2 := map[string]string{
+		api.PreferAvoidPodsAnnotationKey: `
+							{
+							    "preferAvoidPods": [
+							        {
+							            "podSignature": {
+							                "podController": {
+							                    "apiVersion": "v1",
+							                    "kind": "ReplicaSet",
+							                    "name": "foo",
+							                    "uid": "qwert12345",
+							                    "controller": true
+							                }
+							            },
+							            "reason": "some reason",
+							            "message": "some message"
+							        }
+							    ]
+							}`,
+	}
+	testNodes := []*api.Node{
+		{
+			ObjectMeta: api.ObjectMeta{Name: "machine1", Annotations: annotations1},
+		},
+		{
+			ObjectMeta: api.ObjectMeta{Name: "machine2", Annotations: annotations2},
+		},
+		{
+			ObjectMeta: api.ObjectMeta{Name: "machine3"},
+		},
+	}
+	tests := []struct {
+		pod          *api.Pod
+		rcs          []api.ReplicationController
+		rss          []extensions.ReplicaSet
+		nodes        []*api.Node
+		expectedList schedulerapi.HostPriorityList
+		test         string
+	}{
+		{
+			pod: &api.Pod{ObjectMeta: api.ObjectMeta{Namespace: "default", Labels: label1}},
+			rcs: []api.ReplicationController{
+				{
+					ObjectMeta: api.ObjectMeta{
+						Namespace: "default",
+						Name:      "foo",
+						UID:       "abcdef123456",
+					},
+					Spec: api.ReplicationControllerSpec{Selector: label1},
+				},
+			},
+			nodes:        testNodes,
+			expectedList: []schedulerapi.HostPriority{{"machine1", 0}, {"machine2", 10}, {"machine3", 10}},
+			test:         "pod managed by ReplicationController should avoid a node, this node get lowest priority score",
+		},
+		{
+			pod: &api.Pod{ObjectMeta: api.ObjectMeta{Namespace: "default", Labels: label2}},
+			rss: []extensions.ReplicaSet{
+				{
+					TypeMeta: unversioned.TypeMeta{
+						APIVersion: "v1",
+						Kind:       "ReplicaSet",
+					},
+					ObjectMeta: api.ObjectMeta{
+						Namespace: "default",
+						Name:      "bar",
+						UID:       "qwert12345",
+					},
+					Spec: extensions.ReplicaSetSpec{Selector: &unversioned.LabelSelector{MatchLabels: label2}},
+				},
+			},
+			nodes:        testNodes,
+			expectedList: []schedulerapi.HostPriority{{"machine1", 10}, {"machine2", 0}, {"machine3", 10}},
+			test:         "pod managed by ReplicaSet should avoid a node, this node get lowest priority score",
+		},
+		{
+			pod: &api.Pod{ObjectMeta: api.ObjectMeta{Namespace: "default"}},
+			rcs: []api.ReplicationController{
+				{
+					ObjectMeta: api.ObjectMeta{
+						Namespace: "default",
+						Name:      "foo",
+						UID:       "abcdef123456",
+					},
+					Spec: api.ReplicationControllerSpec{Selector: label1},
+				},
+			},
+			nodes:        testNodes,
+			expectedList: []schedulerapi.HostPriority{{"machine1", 10}, {"machine2", 10}, {"machine3", 10}},
+			test:         "pod should not avoid these nodes, all nodes get highest priority score",
+		},
+	}
+
+	for _, test := range tests {
+		prioritizer := NodePreferAvoidPod{
+			controllerLister: algorithm.FakeControllerLister(test.rcs),
+			replicaSetLister: algorithm.FakeReplicaSetLister(test.rss),
+		}
+		list, err := prioritizer.CalculateNodePreferAvoidPodsPriority(test.pod, map[string]*schedulercache.NodeInfo{}, algorithm.FakeNodeLister(test.nodes))
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		// sort the two lists to avoid failures on account of different ordering
+		sort.Sort(test.expectedList)
+		sort.Sort(list)
+		if !reflect.DeepEqual(test.expectedList, list) {
+			t.Errorf("%s: expected %#v, got %#v", test.test, test.expectedList, list)
+		}
+	}
+}

plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go

@@ -170,6 +170,17 @@ func defaultPriorities() sets.String {
 				Weight: 1,
 			},
 		),
+		factory.RegisterPriorityConfigFactory(
+			"NodePreferAvoidPodsPriority",
+			factory.PriorityConfigFactory{
+				Function: func(args factory.PluginFactoryArgs) algorithm.PriorityFunction {
+					return priorities.NewNodePreferAvoidPodsPriority(args.ControllerLister, args.ReplicaSetLister)
+				},
+				// Set this weight large enough to override all other priority functions.
+				// TODO: Figure out a better way to do this, maybe at same time as fixing #24720.
+				Weight: 10000,
+			},
+		),
 		factory.RegisterPriorityFunction("NodeAffinityPriority", priorities.CalculateNodeAffinityPriority, 1),
 		factory.RegisterPriorityFunction("TaintTolerationPriority", priorities.ComputeTaintTolerationPriority, 1),
 	)