Merge pull request #58903 from kubernetes/revert-58555-equivHash

pkg/scheduler/algorithm/predicates/BUILD

@@ -34,6 +34,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
         "//vendor/k8s.io/client-go/listers/storage/v1:go_default_library",

pkg/scheduler/algorithm/predicates/utils.go

@@ -17,8 +17,12 @@ limitations under the License.
 package predicates
 
 import (
+	"github.com/golang/glog"
 	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/pkg/scheduler/algorithm"
 	schedutil "k8s.io/kubernetes/pkg/scheduler/util"
 )
 
@@ -66,6 +70,68 @@ func CreateSelectorFromLabels(aL map[string]string) labels.Selector {
 	return labels.Set(aL).AsSelector()
 }
 
+// EquivalencePodGenerator is a generator of equivalence class for pod with consideration of PVC info.
+type EquivalencePodGenerator struct {
+	pvcInfo PersistentVolumeClaimInfo
+}
+
+// NewEquivalencePodGenerator returns a getEquivalencePod method with consideration of PVC info.
+func NewEquivalencePodGenerator(pvcInfo PersistentVolumeClaimInfo) algorithm.GetEquivalencePodFunc {
+	g := &EquivalencePodGenerator{
+		pvcInfo: pvcInfo,
+	}
+	return g.getEquivalencePod
+}
+
+// GetEquivalencePod returns a EquivalencePod which contains a group of pod attributes which can be reused.
+func (e *EquivalencePodGenerator) getEquivalencePod(pod *v1.Pod) interface{} {
+	// For now we only consider pods:
+	// 1. OwnerReferences is Controller
+	// 2. with same OwnerReferences
+	// 3. with same PVC claim
+	// to be equivalent
+	for _, ref := range pod.OwnerReferences {
+		if ref.Controller != nil && *ref.Controller {
+			if pvcSet, err := e.getPVCSet(pod); err == nil {
+				// A pod can only belongs to one controller, so let's return.
+				return &EquivalencePod{
+					ControllerRef: ref,
+					PVCSet:        pvcSet,
+				}
+			} else {
+				// If error encountered, log warning and return nil (i.e. no equivalent pod found)
+				glog.Warningf("[EquivalencePodGenerator] for pod: %v failed due to: %v", pod.GetName(), err)
+				return nil
+			}
+		}
+	}
+	return nil
+}
+
+// getPVCSet returns a set of PVC UIDs of given pod.
+func (e *EquivalencePodGenerator) getPVCSet(pod *v1.Pod) (sets.String, error) {
+	result := sets.NewString()
+	for _, volume := range pod.Spec.Volumes {
+		if volume.PersistentVolumeClaim == nil {
+			continue
+		}
+		pvcName := volume.PersistentVolumeClaim.ClaimName
+		pvc, err := e.pvcInfo.GetPersistentVolumeClaimInfo(pod.GetNamespace(), pvcName)
+		if err != nil {
+			return nil, err
+		}
+		result.Insert(string(pvc.UID))
+	}
+
+	return result, nil
+}
+
+// EquivalencePod is a group of pod attributes which can be reused as equivalence to schedule other pods.
+type EquivalencePod struct {
+	ControllerRef metav1.OwnerReference
+	PVCSet        sets.String
+}
+
 // portsConflict check whether existingPorts and wantPorts conflict with each other
 // return true if we have a conflict
 func portsConflict(existingPorts schedutil.HostPortInfo, wantPorts []*v1.ContainerPort) bool {

pkg/scheduler/algorithm/types.go

@@ -75,6 +75,8 @@ type PredicateFailureReason interface {
 	GetReason() string
 }
 
+type GetEquivalencePodFunc func(pod *v1.Pod) interface{}
+
 // NodeLister interface represents anything that can list nodes for a scheduler.
 type NodeLister interface {
 	// We explicitly return []*v1.Node, instead of v1.NodeList, to avoid

pkg/scheduler/algorithmprovider/defaults/defaults.go

@@ -77,6 +77,13 @@ func init() {
 	// Fit is determined by node selector query.
 	factory.RegisterFitPredicate(predicates.MatchNodeSelectorPred, predicates.PodMatchNodeSelector)
 
+	// Use equivalence class to speed up heavy predicates phase.
+	factory.RegisterGetEquivalencePodFunction(
+		func(args factory.PluginFactoryArgs) algorithm.GetEquivalencePodFunc {
+			return predicates.NewEquivalencePodGenerator(args.PVCInfo)
+		},
+	)
+
 	// ServiceSpreadingPriority is a priority config factory that spreads pods by minimizing
 	// the number of pods (belonging to the same service) on the same node.
 	// Register the factory so that it's available, but do not include it as part of the default priorities

pkg/scheduler/core/equivalence_cache.go

@@ -37,7 +37,8 @@ const maxCacheEntries = 100
 // 2. function to get equivalence pod
 type EquivalenceCache struct {
 	sync.RWMutex
-	algorithmCache map[string]AlgorithmCache
+	getEquivalencePod algorithm.GetEquivalencePodFunc
+	algorithmCache    map[string]AlgorithmCache
 }
 
 // The AlgorithmCache stores PredicateMap with predicate name as key
@@ -61,9 +62,10 @@ func newAlgorithmCache() AlgorithmCache {
 	}
 }
 
-func NewEquivalenceCache() *EquivalenceCache {
+func NewEquivalenceCache(getEquivalencePodFunc algorithm.GetEquivalencePodFunc) *EquivalenceCache {
 	return &EquivalenceCache{
-		algorithmCache: make(map[string]AlgorithmCache),
+		getEquivalencePod: getEquivalencePodFunc,
+		algorithmCache:    make(map[string]AlgorithmCache),
 	}
 }
 
@@ -206,69 +208,15 @@ func (ec *EquivalenceCache) InvalidateCachedPredicateItemForPodAdd(pod *v1.Pod,
 	ec.InvalidateCachedPredicateItem(nodeName, invalidPredicates)
 }
 
-// getEquivalenceHash computes a hash of the given pod.
-// The hashing function returns the same value for any two pods that are
-// equivalent from the perspective of scheduling.
-func (ec *EquivalenceCache) getEquivalenceHash(pod *v1.Pod) (uint64, bool) {
-	equivalencePod := getEquivalencePod(pod)
-	hash := fnv.New32a()
-	hashutil.DeepHashObject(hash, equivalencePod)
-	return uint64(hash.Sum32()), true
-}
-
-// equivalencePod is the set of pod attributes which must match for two pods to
-// be considered equivalent for scheduling purposes. For correctness, this must
-// include any Pod field which is used by a FitPredicate.
-//
-// NOTE: For equivalence hash to be formally correct, lists and maps in the
-// equivalencePod should be normalized. (e.g. by sorting them) However, the
-// vast majority of equivalent pod classes are expected to be created from a
-// single pod template, so they will all have the same ordering.
-type equivalencePod struct {
-	Namespace      *string
-	Labels         map[string]string
-	Affinity       *v1.Affinity
-	Containers     []v1.Container // See note about ordering
-	InitContainers []v1.Container // See note about ordering
-	NodeName       *string
-	NodeSelector   map[string]string
-	Tolerations    []v1.Toleration
-	Volumes        []v1.Volume // See note about ordering
-}
-
-// getEquivalencePod returns the equivalencePod for a Pod.
-func getEquivalencePod(pod *v1.Pod) *equivalencePod {
-	ep := &equivalencePod{
-		Namespace:      &pod.Namespace,
-		Labels:         pod.Labels,
-		Affinity:       pod.Spec.Affinity,
-		Containers:     pod.Spec.Containers,
-		InitContainers: pod.Spec.InitContainers,
-		NodeName:       &pod.Spec.NodeName,
-		NodeSelector:   pod.Spec.NodeSelector,
-		Tolerations:    pod.Spec.Tolerations,
-		Volumes:        pod.Spec.Volumes,
-	}
-	// DeepHashObject considers nil and empy slices to be different. Normalize them.
-	if len(ep.Containers) == 0 {
-		ep.Containers = nil
-	}
-	if len(ep.InitContainers) == 0 {
-		ep.InitContainers = nil
-	}
-	if len(ep.Tolerations) == 0 {
-		ep.Tolerations = nil
-	}
-	if len(ep.Volumes) == 0 {
-		ep.Volumes = nil
-	}
-	// Normalize empty maps also.
-	if len(ep.Labels) == 0 {
-		ep.Labels = nil
-	}
-	if len(ep.NodeSelector) == 0 {
-		ep.NodeSelector = nil
-	}
-	// TODO: Also normalize nested maps and slices.
-	return ep
+// getHashEquivalencePod returns the hash of equivalence pod.
+// 1. equivalenceHash
+// 2. if equivalence hash is valid
+func (ec *EquivalenceCache) getHashEquivalencePod(pod *v1.Pod) (uint64, bool) {
+	equivalencePod := ec.getEquivalencePod(pod)
+	if equivalencePod != nil {
+		hash := fnv.New32a()
+		hashutil.DeepHashObject(hash, equivalencePod)
+		return uint64(hash.Sum32()), true
+	}
+	return 0, false
 }

pkg/scheduler/core/generic_scheduler.go

@@ -418,7 +418,7 @@ func podFitsOnNode(
 
 	if ecache != nil {
 		// getHashEquivalencePod will return immediately if no equivalence pod found
-		equivalenceHash, eCacheAvailable = ecache.getEquivalenceHash(pod)
+		equivalenceHash, eCacheAvailable = ecache.getHashEquivalencePod(pod)
 	}
 	podsAdded := false
 	// We run predicates twice in some cases. If the node has greater or equal priority

pkg/scheduler/factory/factory.go

@@ -135,8 +135,6 @@ type configFactory struct {
 	alwaysCheckAllPredicates bool
 }
 
-var _ scheduler.Configurator = &configFactory{}
-
 // NewConfigFactory initializes the default implementation of a Configurator To encourage eventual privatization of the struct type, we only
 // return the interface.
 func NewConfigFactory(
@@ -969,8 +967,14 @@ func (f *configFactory) CreateFromKeys(predicateKeys, priorityKeys sets.String,
 	}
 
 	// Init equivalence class cache
-	if f.enableEquivalenceClassCache {
-		f.equivalencePodCache = core.NewEquivalenceCache()
+	if f.enableEquivalenceClassCache && getEquivalencePodFuncFactory != nil {
+		pluginArgs, err := f.getPluginArgs()
+		if err != nil {
+			return nil, err
+		}
+		f.equivalencePodCache = core.NewEquivalenceCache(
+			getEquivalencePodFuncFactory(*pluginArgs),
+		)
 		glog.Info("Created equivalence class cache")
 	}
 

pkg/scheduler/factory/plugins.go

@@ -75,6 +75,9 @@ type PriorityConfigFactory struct {
 	Weight            int
 }
 
+// EquivalencePodFuncFactory produces a function to get equivalence class for given pod.
+type EquivalencePodFuncFactory func(PluginFactoryArgs) algorithm.GetEquivalencePodFunc
+
 var (
 	schedulerFactoryMutex sync.Mutex
 
@@ -87,6 +90,9 @@ var (
 	// Registered metadata producers
 	priorityMetadataProducer  PriorityMetadataProducerFactory
 	predicateMetadataProducer PredicateMetadataProducerFactory
+
+	// get equivalence pod function
+	getEquivalencePodFuncFactory EquivalencePodFuncFactory
 )
 
 const (
@@ -335,6 +341,11 @@ func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy) string {
 	return RegisterPriorityConfigFactory(policy.Name, *pcf)
 }
 
+// RegisterGetEquivalencePodFunction registers equivalenceFuncFactory to produce equivalence class for given pod.
+func RegisterGetEquivalencePodFunction(equivalenceFuncFactory EquivalencePodFuncFactory) {
+	getEquivalencePodFuncFactory = equivalenceFuncFactory
+}
+
 // IsPriorityFunctionRegistered is useful for testing providers.
 func IsPriorityFunctionRegistered(name string) bool {
 	schedulerFactoryMutex.Lock()