Unverified Commit b97092ca authored by Kubernetes Prow Robot's avatar Kubernetes Prow Robot Committed by GitHub

Merge pull request #71399 from resouer/eclass-task-1

Eclass task 1: clean up old code
parents 3dbf0abd 797601d9
...@@ -187,7 +187,6 @@ func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error ...@@ -187,7 +187,6 @@ func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error
stopCh, stopCh,
scheduler.WithName(cc.ComponentConfig.SchedulerName), scheduler.WithName(cc.ComponentConfig.SchedulerName),
scheduler.WithHardPodAffinitySymmetricWeight(cc.ComponentConfig.HardPodAffinitySymmetricWeight), scheduler.WithHardPodAffinitySymmetricWeight(cc.ComponentConfig.HardPodAffinitySymmetricWeight),
scheduler.WithEquivalenceClassCacheEnabled(cc.ComponentConfig.EnableContentionProfiling),
scheduler.WithPreemptionDisabled(cc.ComponentConfig.DisablePreemption), scheduler.WithPreemptionDisabled(cc.ComponentConfig.DisablePreemption),
scheduler.WithPercentageOfNodesToScore(cc.ComponentConfig.PercentageOfNodesToScore), scheduler.WithPercentageOfNodesToScore(cc.ComponentConfig.PercentageOfNodesToScore),
scheduler.WithBindTimeoutSeconds(*cc.ComponentConfig.BindTimeoutSeconds)) scheduler.WithBindTimeoutSeconds(*cc.ComponentConfig.BindTimeoutSeconds))
...@@ -356,7 +355,6 @@ func NewSchedulerConfig(s schedulerserverconfig.CompletedConfig) (*factory.Confi ...@@ -356,7 +355,6 @@ func NewSchedulerConfig(s schedulerserverconfig.CompletedConfig) (*factory.Confi
PdbInformer: s.InformerFactory.Policy().V1beta1().PodDisruptionBudgets(), PdbInformer: s.InformerFactory.Policy().V1beta1().PodDisruptionBudgets(),
StorageClassInformer: storageClassInformer, StorageClassInformer: storageClassInformer,
HardPodAffinitySymmetricWeight: s.ComponentConfig.HardPodAffinitySymmetricWeight, HardPodAffinitySymmetricWeight: s.ComponentConfig.HardPodAffinitySymmetricWeight,
EnableEquivalenceClassCache: utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache),
DisablePreemption: s.ComponentConfig.DisablePreemption, DisablePreemption: s.ComponentConfig.DisablePreemption,
PercentageOfNodesToScore: s.ComponentConfig.PercentageOfNodesToScore, PercentageOfNodesToScore: s.ComponentConfig.PercentageOfNodesToScore,
BindTimeoutSeconds: *s.ComponentConfig.BindTimeoutSeconds, BindTimeoutSeconds: *s.ComponentConfig.BindTimeoutSeconds,
......
...@@ -124,12 +124,6 @@ const ( ...@@ -124,12 +124,6 @@ const (
// Add priority to pods. Priority affects scheduling and preemption of pods. // Add priority to pods. Priority affects scheduling and preemption of pods.
PodPriority utilfeature.Feature = "PodPriority" PodPriority utilfeature.Feature = "PodPriority"
// owner: @resouer
// alpha: v1.8
//
// Enable equivalence class cache for scheduler.
EnableEquivalenceClassCache utilfeature.Feature = "EnableEquivalenceClassCache"
// owner: @k82cn // owner: @k82cn
// beta: v1.12 // beta: v1.12
// //
...@@ -418,7 +412,6 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS ...@@ -418,7 +412,6 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
DebugContainers: {Default: false, PreRelease: utilfeature.Alpha}, DebugContainers: {Default: false, PreRelease: utilfeature.Alpha},
PodShareProcessNamespace: {Default: true, PreRelease: utilfeature.Beta}, PodShareProcessNamespace: {Default: true, PreRelease: utilfeature.Beta},
PodPriority: {Default: true, PreRelease: utilfeature.Beta}, PodPriority: {Default: true, PreRelease: utilfeature.Beta},
EnableEquivalenceClassCache: {Default: false, PreRelease: utilfeature.Alpha},
TaintNodesByCondition: {Default: true, PreRelease: utilfeature.Beta}, TaintNodesByCondition: {Default: true, PreRelease: utilfeature.Beta},
MountPropagation: {Default: true, PreRelease: utilfeature.GA}, MountPropagation: {Default: true, PreRelease: utilfeature.GA},
QOSReserved: {Default: false, PreRelease: utilfeature.Alpha}, QOSReserved: {Default: false, PreRelease: utilfeature.Alpha},
......
...@@ -11,7 +11,6 @@ go_library( ...@@ -11,7 +11,6 @@ go_library(
deps = [ deps = [
"//pkg/features:go_default_library", "//pkg/features:go_default_library",
"//pkg/scheduler/algorithm:go_default_library", "//pkg/scheduler/algorithm:go_default_library",
"//pkg/scheduler/algorithm/predicates:go_default_library",
"//pkg/scheduler/api:go_default_library", "//pkg/scheduler/api:go_default_library",
"//pkg/scheduler/api/latest:go_default_library", "//pkg/scheduler/api/latest:go_default_library",
"//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/apis/config:go_default_library",
......
...@@ -36,8 +36,6 @@ import ( ...@@ -36,8 +36,6 @@ import (
"k8s.io/kubernetes/pkg/scheduler/factory" "k8s.io/kubernetes/pkg/scheduler/factory"
) )
const enableEquivalenceCache = true
func TestCompatibility_v1_Scheduler(t *testing.T) { func TestCompatibility_v1_Scheduler(t *testing.T) {
// Add serialized versions of scheduler config that exercise available options to ensure compatibility between releases // Add serialized versions of scheduler config that exercise available options to ensure compatibility between releases
schedulerFiles := map[string]struct { schedulerFiles := map[string]struct {
...@@ -987,7 +985,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { ...@@ -987,7 +985,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
PdbInformer: informerFactory.Policy().V1beta1().PodDisruptionBudgets(), PdbInformer: informerFactory.Policy().V1beta1().PodDisruptionBudgets(),
StorageClassInformer: informerFactory.Storage().V1().StorageClasses(), StorageClassInformer: informerFactory.Storage().V1().StorageClasses(),
HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight, HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight,
EnableEquivalenceClassCache: enableEquivalenceCache,
DisablePreemption: false, DisablePreemption: false,
PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore,
}).CreateFromConfig(policy); err != nil { }).CreateFromConfig(policy); err != nil {
......
...@@ -49,13 +49,6 @@ func init() { ...@@ -49,13 +49,6 @@ func init() {
registerAlgorithmProvider(defaultPredicates(), defaultPriorities()) registerAlgorithmProvider(defaultPredicates(), defaultPriorities())
// IMPORTANT NOTES for predicate developers: // IMPORTANT NOTES for predicate developers:
// We are using cached predicate result for pods belonging to the same equivalence class.
// So when implementing a new predicate, you are expected to check whether the result
// of your predicate function can be affected by related API object change (ADD/DELETE/UPDATE).
// If yes, you are expected to invalidate the cached predicate result for related API object change.
// For example:
// https://github.com/kubernetes/kubernetes/blob/36a218e/plugin/pkg/scheduler/factory/factory.go#L422
// Registers predicates and priorities that are not enabled by default, but user can pick when creating their // Registers predicates and priorities that are not enabled by default, but user can pick when creating their
// own set of priorities/predicates. // own set of priorities/predicates.
......
...@@ -12,7 +12,6 @@ go_library( ...@@ -12,7 +12,6 @@ go_library(
"//pkg/scheduler/algorithm:go_default_library", "//pkg/scheduler/algorithm:go_default_library",
"//pkg/scheduler/algorithm/predicates:go_default_library", "//pkg/scheduler/algorithm/predicates:go_default_library",
"//pkg/scheduler/api:go_default_library", "//pkg/scheduler/api:go_default_library",
"//pkg/scheduler/core/equivalence:go_default_library",
"//pkg/scheduler/internal/cache:go_default_library", "//pkg/scheduler/internal/cache:go_default_library",
"//pkg/scheduler/internal/queue:go_default_library", "//pkg/scheduler/internal/queue:go_default_library",
"//pkg/scheduler/metrics:go_default_library", "//pkg/scheduler/metrics:go_default_library",
...@@ -48,7 +47,6 @@ go_test( ...@@ -48,7 +47,6 @@ go_test(
"//pkg/scheduler/algorithm/priorities:go_default_library", "//pkg/scheduler/algorithm/priorities:go_default_library",
"//pkg/scheduler/algorithm/priorities/util:go_default_library", "//pkg/scheduler/algorithm/priorities/util:go_default_library",
"//pkg/scheduler/api:go_default_library", "//pkg/scheduler/api:go_default_library",
"//pkg/scheduler/core/equivalence:go_default_library",
"//pkg/scheduler/internal/cache:go_default_library", "//pkg/scheduler/internal/cache:go_default_library",
"//pkg/scheduler/internal/queue:go_default_library", "//pkg/scheduler/internal/queue:go_default_library",
"//pkg/scheduler/nodeinfo:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library",
...@@ -75,10 +73,7 @@ filegroup( ...@@ -75,10 +73,7 @@ filegroup(
filegroup( filegroup(
name = "all-srcs", name = "all-srcs",
srcs = [ srcs = [":package-srcs"],
":package-srcs",
"//pkg/scheduler/core/equivalence:all-srcs",
],
tags = ["automanaged"], tags = ["automanaged"],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = ["eqivalence.go"],
importpath = "k8s.io/kubernetes/pkg/scheduler/core/equivalence",
visibility = ["//visibility:public"],
deps = [
"//pkg/features:go_default_library",
"//pkg/scheduler/algorithm:go_default_library",
"//pkg/scheduler/algorithm/predicates:go_default_library",
"//pkg/scheduler/metrics:go_default_library",
"//pkg/scheduler/nodeinfo:go_default_library",
"//pkg/util/hash:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["eqivalence_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/scheduler/algorithm:go_default_library",
"//pkg/scheduler/algorithm/predicates:go_default_library",
"//pkg/scheduler/nodeinfo:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
...@@ -510,7 +510,6 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { ...@@ -510,7 +510,6 @@ func TestGenericSchedulerWithExtenders(t *testing.T) {
queue := internalqueue.NewSchedulingQueue(nil) queue := internalqueue.NewSchedulingQueue(nil)
scheduler := NewGenericScheduler( scheduler := NewGenericScheduler(
cache, cache,
nil,
queue, queue,
test.predicates, test.predicates,
algorithm.EmptyPredicateMetadataProducer, algorithm.EmptyPredicateMetadataProducer,
......
...@@ -39,7 +39,6 @@ import ( ...@@ -39,7 +39,6 @@ import (
"k8s.io/kubernetes/pkg/scheduler/algorithm" "k8s.io/kubernetes/pkg/scheduler/algorithm"
"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
"k8s.io/kubernetes/pkg/scheduler/core/equivalence"
schedulerinternalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" schedulerinternalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue"
"k8s.io/kubernetes/pkg/scheduler/metrics" "k8s.io/kubernetes/pkg/scheduler/metrics"
...@@ -116,7 +115,6 @@ type ScheduleAlgorithm interface { ...@@ -116,7 +115,6 @@ type ScheduleAlgorithm interface {
type genericScheduler struct { type genericScheduler struct {
cache schedulerinternalcache.Cache cache schedulerinternalcache.Cache
equivalenceCache *equivalence.Cache
schedulingQueue internalqueue.SchedulingQueue schedulingQueue internalqueue.SchedulingQueue
predicates map[string]algorithm.FitPredicate predicates map[string]algorithm.FitPredicate
priorityMetaProducer algorithm.PriorityMetadataProducer priorityMetaProducer algorithm.PriorityMetadataProducer
...@@ -134,21 +132,9 @@ type genericScheduler struct { ...@@ -134,21 +132,9 @@ type genericScheduler struct {
percentageOfNodesToScore int32 percentageOfNodesToScore int32
} }
// snapshot snapshots equivalence cache and node infos for all fit and priority // snapshot snapshots scheduler cache and node infos for all fit and priority
// functions. // functions.
func (g *genericScheduler) snapshot() error { func (g *genericScheduler) snapshot() error {
// IMPORTANT NOTE: We must snapshot equivalence cache before snapshotting
// scheduler cache, otherwise stale data may be written into equivalence
// cache, e.g.
// 1. snapshot cache
// 2. event arrives, updating cache and invalidating predicates or whole node cache
// 3. snapshot ecache
// 4. evaluate predicates
// 5. stale result will be written to ecache
if g.equivalenceCache != nil {
g.equivalenceCache.Snapshot()
}
// Used for all fit and priority funcs. // Used for all fit and priority funcs.
return g.cache.UpdateNodeNameToInfoMap(g.cachedNodeInfoMap) return g.cache.UpdateNodeNameToInfoMap(g.cachedNodeInfoMap)
} }
...@@ -420,7 +406,6 @@ func (g *genericScheduler) findNodesThatFit(pod *v1.Pod, nodes []*v1.Node) ([]*v ...@@ -420,7 +406,6 @@ func (g *genericScheduler) findNodesThatFit(pod *v1.Pod, nodes []*v1.Node) ([]*v
var ( var (
predicateResultLock sync.Mutex predicateResultLock sync.Mutex
filteredLen int32 filteredLen int32
equivClass *equivalence.Class
) )
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
...@@ -428,26 +413,15 @@ func (g *genericScheduler) findNodesThatFit(pod *v1.Pod, nodes []*v1.Node) ([]*v ...@@ -428,26 +413,15 @@ func (g *genericScheduler) findNodesThatFit(pod *v1.Pod, nodes []*v1.Node) ([]*v
// We can use the same metadata producer for all nodes. // We can use the same metadata producer for all nodes.
meta := g.predicateMetaProducer(pod, g.cachedNodeInfoMap) meta := g.predicateMetaProducer(pod, g.cachedNodeInfoMap)
if g.equivalenceCache != nil {
// getEquivalenceClassInfo will return immediately if no equivalence pod found
equivClass = equivalence.NewClass(pod)
}
checkNode := func(i int) { checkNode := func(i int) {
var nodeCache *equivalence.NodeCache
nodeName := g.cache.NodeTree().Next() nodeName := g.cache.NodeTree().Next()
if g.equivalenceCache != nil {
nodeCache = g.equivalenceCache.LoadNodeCache(nodeName)
}
fits, failedPredicates, err := podFitsOnNode( fits, failedPredicates, err := podFitsOnNode(
pod, pod,
meta, meta,
g.cachedNodeInfoMap[nodeName], g.cachedNodeInfoMap[nodeName],
g.predicates, g.predicates,
nodeCache,
g.schedulingQueue, g.schedulingQueue,
g.alwaysCheckAllPredicates, g.alwaysCheckAllPredicates,
equivClass,
) )
if err != nil { if err != nil {
predicateResultLock.Lock() predicateResultLock.Lock()
...@@ -556,15 +530,10 @@ func podFitsOnNode( ...@@ -556,15 +530,10 @@ func podFitsOnNode(
meta algorithm.PredicateMetadata, meta algorithm.PredicateMetadata,
info *schedulernodeinfo.NodeInfo, info *schedulernodeinfo.NodeInfo,
predicateFuncs map[string]algorithm.FitPredicate, predicateFuncs map[string]algorithm.FitPredicate,
nodeCache *equivalence.NodeCache,
queue internalqueue.SchedulingQueue, queue internalqueue.SchedulingQueue,
alwaysCheckAllPredicates bool, alwaysCheckAllPredicates bool,
equivClass *equivalence.Class,
) (bool, []algorithm.PredicateFailureReason, error) { ) (bool, []algorithm.PredicateFailureReason, error) {
var ( var failedPredicates []algorithm.PredicateFailureReason
eCacheAvailable bool
failedPredicates []algorithm.PredicateFailureReason
)
podsAdded := false podsAdded := false
// We run predicates twice in some cases. If the node has greater or equal priority // We run predicates twice in some cases. If the node has greater or equal priority
...@@ -593,11 +562,7 @@ func podFitsOnNode( ...@@ -593,11 +562,7 @@ func podFitsOnNode(
} else if !podsAdded || len(failedPredicates) != 0 { } else if !podsAdded || len(failedPredicates) != 0 {
break break
} }
// Bypass eCache if node has any nominated pods. for _, predicateKey := range predicates.Ordering() {
// TODO(bsalamat): consider using eCache and adding proper eCache invalidations
// when pods are nominated or their nominations change.
eCacheAvailable = equivClass != nil && nodeCache != nil && !podsAdded
for predicateID, predicateKey := range predicates.Ordering() {
var ( var (
fit bool fit bool
reasons []algorithm.PredicateFailureReason reasons []algorithm.PredicateFailureReason
...@@ -605,11 +570,7 @@ func podFitsOnNode( ...@@ -605,11 +570,7 @@ func podFitsOnNode(
) )
//TODO (yastij) : compute average predicate restrictiveness to export it as Prometheus metric //TODO (yastij) : compute average predicate restrictiveness to export it as Prometheus metric
if predicate, exist := predicateFuncs[predicateKey]; exist { if predicate, exist := predicateFuncs[predicateKey]; exist {
if eCacheAvailable { fit, reasons, err = predicate(pod, metaToUse, nodeInfoToUse)
fit, reasons, err = nodeCache.RunPredicate(predicate, predicateKey, predicateID, pod, metaToUse, nodeInfoToUse, equivClass)
} else {
fit, reasons, err = predicate(pod, metaToUse, nodeInfoToUse)
}
if err != nil { if err != nil {
return false, []algorithm.PredicateFailureReason{}, err return false, []algorithm.PredicateFailureReason{}, err
} }
...@@ -1038,7 +999,7 @@ func selectVictimsOnNode( ...@@ -1038,7 +999,7 @@ func selectVictimsOnNode(
// that we should check is if the "pod" is failing to schedule due to pod affinity // that we should check is if the "pod" is failing to schedule due to pod affinity
// failure. // failure.
// TODO(bsalamat): Consider checking affinity to lower priority pods if feasible with reasonable performance. // TODO(bsalamat): Consider checking affinity to lower priority pods if feasible with reasonable performance.
if fits, _, err := podFitsOnNode(pod, meta, nodeInfoCopy, fitPredicates, nil, queue, false, nil); !fits { if fits, _, err := podFitsOnNode(pod, meta, nodeInfoCopy, fitPredicates, queue, false); !fits {
if err != nil { if err != nil {
klog.Warningf("Encountered error while selecting victims on node %v: %v", nodeInfo.Node().Name, err) klog.Warningf("Encountered error while selecting victims on node %v: %v", nodeInfo.Node().Name, err)
} }
...@@ -1052,7 +1013,7 @@ func selectVictimsOnNode( ...@@ -1052,7 +1013,7 @@ func selectVictimsOnNode(
violatingVictims, nonViolatingVictims := filterPodsWithPDBViolation(potentialVictims.Items, pdbs) violatingVictims, nonViolatingVictims := filterPodsWithPDBViolation(potentialVictims.Items, pdbs)
reprievePod := func(p *v1.Pod) bool { reprievePod := func(p *v1.Pod) bool {
addPod(p) addPod(p)
fits, _, _ := podFitsOnNode(pod, meta, nodeInfoCopy, fitPredicates, nil, queue, false, nil) fits, _, _ := podFitsOnNode(pod, meta, nodeInfoCopy, fitPredicates, queue, false)
if !fits { if !fits {
removePod(p) removePod(p)
victims = append(victims, p) victims = append(victims, p)
...@@ -1168,7 +1129,6 @@ func podPassesBasicChecks(pod *v1.Pod, pvcLister corelisters.PersistentVolumeCla ...@@ -1168,7 +1129,6 @@ func podPassesBasicChecks(pod *v1.Pod, pvcLister corelisters.PersistentVolumeCla
// NewGenericScheduler creates a genericScheduler object. // NewGenericScheduler creates a genericScheduler object.
func NewGenericScheduler( func NewGenericScheduler(
cache schedulerinternalcache.Cache, cache schedulerinternalcache.Cache,
eCache *equivalence.Cache,
podQueue internalqueue.SchedulingQueue, podQueue internalqueue.SchedulingQueue,
predicates map[string]algorithm.FitPredicate, predicates map[string]algorithm.FitPredicate,
predicateMetaProducer algorithm.PredicateMetadataProducer, predicateMetaProducer algorithm.PredicateMetadataProducer,
...@@ -1185,7 +1145,6 @@ func NewGenericScheduler( ...@@ -1185,7 +1145,6 @@ func NewGenericScheduler(
) ScheduleAlgorithm { ) ScheduleAlgorithm {
return &genericScheduler{ return &genericScheduler{
cache: cache, cache: cache,
equivalenceCache: eCache,
schedulingQueue: podQueue, schedulingQueue: podQueue,
predicates: predicates, predicates: predicates,
predicateMetaProducer: predicateMetaProducer, predicateMetaProducer: predicateMetaProducer,
......
...@@ -22,7 +22,6 @@ import ( ...@@ -22,7 +22,6 @@ import (
"reflect" "reflect"
"strconv" "strconv"
"strings" "strings"
"sync"
"testing" "testing"
"time" "time"
...@@ -39,7 +38,6 @@ import ( ...@@ -39,7 +38,6 @@ import (
algorithmpriorities "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" algorithmpriorities "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities"
priorityutil "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/util" priorityutil "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/util"
schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
"k8s.io/kubernetes/pkg/scheduler/core/equivalence"
schedulerinternalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" schedulerinternalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue"
schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
...@@ -471,7 +469,6 @@ func TestGenericScheduler(t *testing.T) { ...@@ -471,7 +469,6 @@ func TestGenericScheduler(t *testing.T) {
scheduler := NewGenericScheduler( scheduler := NewGenericScheduler(
cache, cache,
nil,
internalqueue.NewSchedulingQueue(nil), internalqueue.NewSchedulingQueue(nil),
test.predicates, test.predicates,
algorithm.EmptyPredicateMetadataProducer, algorithm.EmptyPredicateMetadataProducer,
...@@ -508,7 +505,6 @@ func makeScheduler(predicates map[string]algorithm.FitPredicate, nodes []*v1.Nod ...@@ -508,7 +505,6 @@ func makeScheduler(predicates map[string]algorithm.FitPredicate, nodes []*v1.Nod
s := NewGenericScheduler( s := NewGenericScheduler(
cache, cache,
nil,
internalqueue.NewSchedulingQueue(nil), internalqueue.NewSchedulingQueue(nil),
predicates, predicates,
algorithm.EmptyPredicateMetadataProducer, algorithm.EmptyPredicateMetadataProducer,
...@@ -1435,7 +1431,6 @@ func TestPreempt(t *testing.T) { ...@@ -1435,7 +1431,6 @@ func TestPreempt(t *testing.T) {
} }
scheduler := NewGenericScheduler( scheduler := NewGenericScheduler(
cache, cache,
nil,
internalqueue.NewSchedulingQueue(nil), internalqueue.NewSchedulingQueue(nil),
map[string]algorithm.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, map[string]algorithm.FitPredicate{"matches": algorithmpredicates.PodFitsResources},
algorithm.EmptyPredicateMetadataProducer, algorithm.EmptyPredicateMetadataProducer,
...@@ -1488,189 +1483,3 @@ func TestPreempt(t *testing.T) { ...@@ -1488,189 +1483,3 @@ func TestPreempt(t *testing.T) {
}) })
} }
} }
// syncingMockCache delegates method calls to an actual Cache,
// but calls to UpdateNodeNameToInfoMap synchronize with the test.
type syncingMockCache struct {
schedulerinternalcache.Cache
cycleStart, cacheInvalidated chan struct{}
once sync.Once
}
// UpdateNodeNameToInfoMap delegates to the real implementation, but on the first call, it
// synchronizes with the test.
//
// Since UpdateNodeNameToInfoMap is one of the first steps of (*genericScheduler).Schedule, we use
// this point to signal to the test that a scheduling cycle has started.
func (c *syncingMockCache) UpdateNodeNameToInfoMap(infoMap map[string]*schedulernodeinfo.NodeInfo) error {
err := c.Cache.UpdateNodeNameToInfoMap(infoMap)
c.once.Do(func() {
c.cycleStart <- struct{}{}
<-c.cacheInvalidated
})
return err
}
// TestCacheInvalidationRace tests that equivalence cache invalidation is correctly
// handled when an invalidation event happens early in a scheduling cycle. Specifically, the event
// occurs after schedulernodeinfo is snapshotted and before equivalence cache lock is acquired.
func TestCacheInvalidationRace(t *testing.T) {
// Create a predicate that returns false the first time and true on subsequent calls.
podWillFit := false
var callCount int
testPredicate := func(pod *v1.Pod,
meta algorithm.PredicateMetadata,
nodeInfo *schedulernodeinfo.NodeInfo) (bool, []algorithm.PredicateFailureReason, error) {
callCount++
if !podWillFit {
podWillFit = true
return false, []algorithm.PredicateFailureReason{algorithmpredicates.ErrFakePredicate}, nil
}
return true, nil, nil
}
// Set up the mock cache.
cache := schedulerinternalcache.New(time.Duration(0), wait.NeverStop)
testNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "machine1"}}
cache.AddNode(testNode)
mockCache := &syncingMockCache{
Cache: cache,
cycleStart: make(chan struct{}),
cacheInvalidated: make(chan struct{}),
}
ps := map[string]algorithm.FitPredicate{"testPredicate": testPredicate}
algorithmpredicates.SetPredicatesOrdering([]string{"testPredicate"})
eCache := equivalence.NewCache(algorithmpredicates.Ordering())
eCache.GetNodeCache(testNode.Name)
// Ensure that equivalence cache invalidation happens after the scheduling cycle starts, but before
// the equivalence cache would be updated.
go func() {
<-mockCache.cycleStart
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "new-pod", UID: "new-pod"},
Spec: v1.PodSpec{NodeName: "machine1"}}
if err := cache.AddPod(pod); err != nil {
t.Errorf("Could not add pod to cache: %v", err)
}
eCache.InvalidateAllPredicatesOnNode("machine1")
mockCache.cacheInvalidated <- struct{}{}
}()
// Set up the scheduler.
prioritizers := []algorithm.PriorityConfig{{Map: EqualPriorityMap, Weight: 1}}
pvcLister := schedulertesting.FakePersistentVolumeClaimLister([]*v1.PersistentVolumeClaim{})
pdbLister := schedulertesting.FakePDBLister{}
scheduler := NewGenericScheduler(
mockCache,
eCache,
internalqueue.NewSchedulingQueue(nil),
ps,
algorithm.EmptyPredicateMetadataProducer,
prioritizers,
algorithm.EmptyPriorityMetadataProducer,
emptyPluginSet,
nil, nil, pvcLister, pdbLister,
true, false,
schedulerapi.DefaultPercentageOfNodesToScore)
// First scheduling attempt should fail.
nodeLister := schedulertesting.FakeNodeLister(makeNodeList([]string{"machine1"}))
pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-pod"}}
machine, err := scheduler.Schedule(pod, nodeLister)
if machine != "" || err == nil {
t.Error("First scheduling attempt did not fail")
}
// Second scheduling attempt should succeed because cache was invalidated.
_, err = scheduler.Schedule(pod, nodeLister)
if err != nil {
t.Errorf("Second scheduling attempt failed: %v", err)
}
if callCount != 2 {
t.Errorf("Predicate should have been called twice. Was called %d times.", callCount)
}
}
// TestCacheInvalidationRace2 tests that cache invalidation is correctly handled
// when an invalidation event happens while a predicate is running.
func TestCacheInvalidationRace2(t *testing.T) {
// Create a predicate that returns false the first time and true on subsequent calls.
var (
podWillFit = false
callCount int
cycleStart = make(chan struct{})
cacheInvalidated = make(chan struct{})
once sync.Once
)
testPredicate := func(pod *v1.Pod,
meta algorithm.PredicateMetadata,
nodeInfo *schedulernodeinfo.NodeInfo) (bool, []algorithm.PredicateFailureReason, error) {
callCount++
once.Do(func() {
cycleStart <- struct{}{}
<-cacheInvalidated
})
if !podWillFit {
podWillFit = true
return false, []algorithm.PredicateFailureReason{algorithmpredicates.ErrFakePredicate}, nil
}
return true, nil, nil
}
// Set up the mock cache.
cache := schedulerinternalcache.New(time.Duration(0), wait.NeverStop)
testNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "machine1"}}
cache.AddNode(testNode)
ps := map[string]algorithm.FitPredicate{"testPredicate": testPredicate}
algorithmpredicates.SetPredicatesOrdering([]string{"testPredicate"})
eCache := equivalence.NewCache(algorithmpredicates.Ordering())
eCache.GetNodeCache(testNode.Name)
// Ensure that equivalence cache invalidation happens after the scheduling cycle starts, but before
// the equivalence cache would be updated.
go func() {
<-cycleStart
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "new-pod", UID: "new-pod"},
Spec: v1.PodSpec{NodeName: "machine1"}}
if err := cache.AddPod(pod); err != nil {
t.Errorf("Could not add pod to cache: %v", err)
}
eCache.InvalidateAllPredicatesOnNode("machine1")
cacheInvalidated <- struct{}{}
}()
// Set up the scheduler.
prioritizers := []algorithm.PriorityConfig{{Map: EqualPriorityMap, Weight: 1}}
pvcLister := schedulertesting.FakePersistentVolumeClaimLister([]*v1.PersistentVolumeClaim{})
pdbLister := schedulertesting.FakePDBLister{}
scheduler := NewGenericScheduler(
cache,
eCache,
internalqueue.NewSchedulingQueue(nil),
ps,
algorithm.EmptyPredicateMetadataProducer,
prioritizers,
algorithm.EmptyPriorityMetadataProducer,
emptyPluginSet,
nil, nil, pvcLister, pdbLister, true, false,
schedulerapi.DefaultPercentageOfNodesToScore)
// First scheduling attempt should fail.
nodeLister := schedulertesting.FakeNodeLister(makeNodeList([]string{"machine1"}))
pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-pod"}}
machine, err := scheduler.Schedule(pod, nodeLister)
if machine != "" || err == nil {
t.Error("First scheduling attempt did not fail")
}
// Second scheduling attempt should succeed because cache was invalidated.
_, err = scheduler.Schedule(pod, nodeLister)
if err != nil {
t.Errorf("Second scheduling attempt failed: %v", err)
}
if callCount != 2 {
t.Errorf("Predicate should have been called twice. Was called %d times.", callCount)
}
}
...@@ -12,7 +12,6 @@ go_library( ...@@ -12,7 +12,6 @@ go_library(
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
deps = [ deps = [
"//pkg/api/v1/pod:go_default_library", "//pkg/api/v1/pod:go_default_library",
"//pkg/apis/core/helper:go_default_library",
"//pkg/features:go_default_library", "//pkg/features:go_default_library",
"//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/apis:go_default_library",
"//pkg/scheduler/algorithm:go_default_library", "//pkg/scheduler/algorithm:go_default_library",
...@@ -21,7 +20,6 @@ go_library( ...@@ -21,7 +20,6 @@ go_library(
"//pkg/scheduler/api:go_default_library", "//pkg/scheduler/api:go_default_library",
"//pkg/scheduler/api/validation:go_default_library", "//pkg/scheduler/api/validation:go_default_library",
"//pkg/scheduler/core:go_default_library", "//pkg/scheduler/core:go_default_library",
"//pkg/scheduler/core/equivalence:go_default_library",
"//pkg/scheduler/internal/cache:go_default_library", "//pkg/scheduler/internal/cache:go_default_library",
"//pkg/scheduler/internal/cache/debugger:go_default_library", "//pkg/scheduler/internal/cache/debugger:go_default_library",
"//pkg/scheduler/internal/queue:go_default_library", "//pkg/scheduler/internal/queue:go_default_library",
......
...@@ -45,9 +45,8 @@ import ( ...@@ -45,9 +45,8 @@ import (
) )
const ( const (
enableEquivalenceCache = true disablePodPreemption = false
disablePodPreemption = false bindTimeoutSeconds = 600
bindTimeoutSeconds = 600
) )
func TestCreate(t *testing.T) { func TestCreate(t *testing.T) {
...@@ -532,7 +531,6 @@ func newConfigFactory(client clientset.Interface, hardPodAffinitySymmetricWeight ...@@ -532,7 +531,6 @@ func newConfigFactory(client clientset.Interface, hardPodAffinitySymmetricWeight
informerFactory.Policy().V1beta1().PodDisruptionBudgets(), informerFactory.Policy().V1beta1().PodDisruptionBudgets(),
informerFactory.Storage().V1().StorageClasses(), informerFactory.Storage().V1().StorageClasses(),
hardPodAffinitySymmetricWeight, hardPodAffinitySymmetricWeight,
enableEquivalenceCache,
disablePodPreemption, disablePodPreemption,
schedulerapi.DefaultPercentageOfNodesToScore, schedulerapi.DefaultPercentageOfNodesToScore,
bindTimeoutSeconds, bindTimeoutSeconds,
......
...@@ -131,22 +131,6 @@ var ( ...@@ -131,22 +131,6 @@ var (
Help: "Total preemption attempts in the cluster till now", Help: "Total preemption attempts in the cluster till now",
}) })
equivalenceCacheLookups = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: SchedulerSubsystem,
Name: "equiv_cache_lookups_total",
Help: "Total number of equivalence cache lookups, by whether or not a cache entry was found",
}, []string{"result"})
EquivalenceCacheHits = equivalenceCacheLookups.With(prometheus.Labels{"result": "hit"})
EquivalenceCacheMisses = equivalenceCacheLookups.With(prometheus.Labels{"result": "miss"})
EquivalenceCacheWrites = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: SchedulerSubsystem,
Name: "equiv_cache_writes",
Help: "Total number of equivalence cache writes, by result",
}, []string{"result"})
metricsList = []prometheus.Collector{ metricsList = []prometheus.Collector{
scheduleAttempts, scheduleAttempts,
SchedulingLatency, SchedulingLatency,
...@@ -158,8 +142,6 @@ var ( ...@@ -158,8 +142,6 @@ var (
SchedulingAlgorithmPremptionEvaluationDuration, SchedulingAlgorithmPremptionEvaluationDuration,
PreemptionVictims, PreemptionVictims,
PreemptionAttempts, PreemptionAttempts,
equivalenceCacheLookups,
EquivalenceCacheWrites,
} }
) )
......
...@@ -28,7 +28,6 @@ import ( ...@@ -28,7 +28,6 @@ import (
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeature "k8s.io/apiserver/pkg/util/feature"
appsinformers "k8s.io/client-go/informers/apps/v1" appsinformers "k8s.io/client-go/informers/apps/v1"
...@@ -38,7 +37,6 @@ import ( ...@@ -38,7 +37,6 @@ import (
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
"k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
latestschedulerapi "k8s.io/kubernetes/pkg/scheduler/api/latest" latestschedulerapi "k8s.io/kubernetes/pkg/scheduler/api/latest"
kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
...@@ -70,7 +68,6 @@ func (sched *Scheduler) Cache() schedulerinternalcache.Cache { ...@@ -70,7 +68,6 @@ func (sched *Scheduler) Cache() schedulerinternalcache.Cache {
type schedulerOptions struct { type schedulerOptions struct {
schedulerName string schedulerName string
hardPodAffinitySymmetricWeight int32 hardPodAffinitySymmetricWeight int32
enableEquivalenceClassCache bool
disablePreemption bool disablePreemption bool
percentageOfNodesToScore int32 percentageOfNodesToScore int32
bindTimeoutSeconds int64 bindTimeoutSeconds int64
...@@ -93,13 +90,6 @@ func WithHardPodAffinitySymmetricWeight(hardPodAffinitySymmetricWeight int32) Op ...@@ -93,13 +90,6 @@ func WithHardPodAffinitySymmetricWeight(hardPodAffinitySymmetricWeight int32) Op
} }
} }
// WithEquivalenceClassCacheEnabled sets enableEquivalenceClassCache for Scheduler, the default value is false
func WithEquivalenceClassCacheEnabled(enableEquivalenceClassCache bool) Option {
return func(o *schedulerOptions) {
o.enableEquivalenceClassCache = enableEquivalenceClassCache
}
}
// WithPreemptionDisabled sets disablePreemption for Scheduler, the default value is false // WithPreemptionDisabled sets disablePreemption for Scheduler, the default value is false
func WithPreemptionDisabled(disablePreemption bool) Option { func WithPreemptionDisabled(disablePreemption bool) Option {
return func(o *schedulerOptions) { return func(o *schedulerOptions) {
...@@ -124,7 +114,6 @@ func WithBindTimeoutSeconds(bindTimeoutSeconds int64) Option { ...@@ -124,7 +114,6 @@ func WithBindTimeoutSeconds(bindTimeoutSeconds int64) Option {
var defaultSchedulerOptions = schedulerOptions{ var defaultSchedulerOptions = schedulerOptions{
schedulerName: v1.DefaultSchedulerName, schedulerName: v1.DefaultSchedulerName,
hardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight, hardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceClassCache: false,
disablePreemption: false, disablePreemption: false,
percentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, percentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore,
bindTimeoutSeconds: BindTimeoutSeconds, bindTimeoutSeconds: BindTimeoutSeconds,
...@@ -167,7 +156,6 @@ func New(client clientset.Interface, ...@@ -167,7 +156,6 @@ func New(client clientset.Interface,
PdbInformer: pdbInformer, PdbInformer: pdbInformer,
StorageClassInformer: storageClassInformer, StorageClassInformer: storageClassInformer,
HardPodAffinitySymmetricWeight: options.hardPodAffinitySymmetricWeight, HardPodAffinitySymmetricWeight: options.hardPodAffinitySymmetricWeight,
EnableEquivalenceClassCache: options.enableEquivalenceClassCache,
DisablePreemption: options.disablePreemption, DisablePreemption: options.disablePreemption,
PercentageOfNodesToScore: options.percentageOfNodesToScore, PercentageOfNodesToScore: options.percentageOfNodesToScore,
BindTimeoutSeconds: options.bindTimeoutSeconds, BindTimeoutSeconds: options.bindTimeoutSeconds,
...@@ -370,12 +358,6 @@ func (sched *Scheduler) assumeVolumes(assumed *v1.Pod, host string) (allBound bo ...@@ -370,12 +358,6 @@ func (sched *Scheduler) assumeVolumes(assumed *v1.Pod, host string) (allBound bo
sched.recordSchedulingFailure(assumed, err, SchedulerError, sched.recordSchedulingFailure(assumed, err, SchedulerError,
fmt.Sprintf("AssumePodVolumes failed: %v", err)) fmt.Sprintf("AssumePodVolumes failed: %v", err))
} }
// Invalidate ecache because assumed volumes could have affected the cached
// pvs for other pods
if sched.config.Ecache != nil {
invalidPredicates := sets.NewString(predicates.CheckVolumeBindingPred)
sched.config.Ecache.InvalidatePredicates(invalidPredicates)
}
} }
return return
} }
...@@ -416,11 +398,7 @@ func (sched *Scheduler) assume(assumed *v1.Pod, host string) error { ...@@ -416,11 +398,7 @@ func (sched *Scheduler) assume(assumed *v1.Pod, host string) error {
// If the binding fails, scheduler will release resources allocated to assumed pod // If the binding fails, scheduler will release resources allocated to assumed pod
// immediately. // immediately.
assumed.Spec.NodeName = host assumed.Spec.NodeName = host
// NOTE: Updates must be written to scheduler cache before invalidating
// equivalence cache, because we could snapshot equivalence cache after the
// invalidation and then snapshot the cache itself. If the cache is
// snapshotted before updates are written, we would update equivalence
// cache with stale information which is based on snapshot of old cache.
if err := sched.config.SchedulerCache.AssumePod(assumed); err != nil { if err := sched.config.SchedulerCache.AssumePod(assumed); err != nil {
klog.Errorf("scheduler cache AssumePod failed: %v", err) klog.Errorf("scheduler cache AssumePod failed: %v", err)
...@@ -438,12 +416,6 @@ func (sched *Scheduler) assume(assumed *v1.Pod, host string) error { ...@@ -438,12 +416,6 @@ func (sched *Scheduler) assume(assumed *v1.Pod, host string) error {
sched.config.SchedulingQueue.DeleteNominatedPodIfExists(assumed) sched.config.SchedulingQueue.DeleteNominatedPodIfExists(assumed)
} }
// Optimistically assume that the binding will succeed, so we need to invalidate affected
// predicates in equivalence cache.
// If the binding fails, these invalidated item will not break anything.
if sched.config.Ecache != nil {
sched.config.Ecache.InvalidateCachedPredicateItemForPodAdd(assumed, host)
}
return nil return nil
} }
......
...@@ -639,7 +639,6 @@ func setupTestScheduler(queuedPodStore *clientcache.FIFO, scache schedulerintern ...@@ -639,7 +639,6 @@ func setupTestScheduler(queuedPodStore *clientcache.FIFO, scache schedulerintern
algo := core.NewGenericScheduler( algo := core.NewGenericScheduler(
scache, scache,
nil, nil,
nil,
predicateMap, predicateMap,
algorithm.EmptyPredicateMetadataProducer, algorithm.EmptyPredicateMetadataProducer,
[]algorithm.PriorityConfig{}, []algorithm.PriorityConfig{},
...@@ -692,7 +691,6 @@ func setupTestSchedulerLongBindingWithRetry(queuedPodStore *clientcache.FIFO, sc ...@@ -692,7 +691,6 @@ func setupTestSchedulerLongBindingWithRetry(queuedPodStore *clientcache.FIFO, sc
algo := core.NewGenericScheduler( algo := core.NewGenericScheduler(
scache, scache,
nil, nil,
nil,
predicateMap, predicateMap,
algorithm.EmptyPredicateMetadataProducer, algorithm.EmptyPredicateMetadataProducer,
[]algorithm.PriorityConfig{}, []algorithm.PriorityConfig{},
......
...@@ -112,7 +112,6 @@ func setupScheduler( ...@@ -112,7 +112,6 @@ func setupScheduler(
PdbInformer: informerFactory.Policy().V1beta1().PodDisruptionBudgets(), PdbInformer: informerFactory.Policy().V1beta1().PodDisruptionBudgets(),
StorageClassInformer: informerFactory.Storage().V1().StorageClasses(), StorageClassInformer: informerFactory.Storage().V1().StorageClasses(),
HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight, HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight,
EnableEquivalenceClassCache: false,
DisablePreemption: false, DisablePreemption: false,
PercentageOfNodesToScore: 100, PercentageOfNodesToScore: 100,
}) })
......
...@@ -93,7 +93,6 @@ go_library( ...@@ -93,7 +93,6 @@ go_library(
"//pkg/api/v1/pod:go_default_library", "//pkg/api/v1/pod:go_default_library",
"//pkg/controller:go_default_library", "//pkg/controller:go_default_library",
"//pkg/controller/disruption:go_default_library", "//pkg/controller/disruption:go_default_library",
"//pkg/features:go_default_library",
"//pkg/scheduler:go_default_library", "//pkg/scheduler:go_default_library",
"//pkg/scheduler/algorithmprovider:go_default_library", "//pkg/scheduler/algorithmprovider:go_default_library",
"//pkg/scheduler/api:go_default_library", "//pkg/scheduler/api:go_default_library",
...@@ -110,8 +109,6 @@ go_library( ...@@ -110,8 +109,6 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
"//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library",
"//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
......
...@@ -106,7 +106,7 @@ func TestReservePlugin(t *testing.T) { ...@@ -106,7 +106,7 @@ func TestReservePlugin(t *testing.T) {
// Create the master and the scheduler with the test plugin set. // Create the master and the scheduler with the test plugin set.
context := initTestSchedulerWithOptions(t, context := initTestSchedulerWithOptions(t,
initTestMaster(t, "reserve-plugin", nil), initTestMaster(t, "reserve-plugin", nil),
false, nil, testPluginSet, false, true, time.Second) false, nil, testPluginSet, false, time.Second)
defer cleanupTest(t, context) defer cleanupTest(t, context)
cs := context.clientSet cs := context.clientSet
...@@ -157,7 +157,7 @@ func TestPrebindPlugin(t *testing.T) { ...@@ -157,7 +157,7 @@ func TestPrebindPlugin(t *testing.T) {
// Create the master and the scheduler with the test plugin set. // Create the master and the scheduler with the test plugin set.
context := initTestSchedulerWithOptions(t, context := initTestSchedulerWithOptions(t,
initTestMaster(t, "prebind-plugin", nil), initTestMaster(t, "prebind-plugin", nil),
false, nil, testPluginSet, false, true, time.Second) false, nil, testPluginSet, false, time.Second)
defer cleanupTest(t, context) defer cleanupTest(t, context)
cs := context.clientSet cs := context.clientSet
...@@ -239,7 +239,7 @@ func TestContextCleanup(t *testing.T) { ...@@ -239,7 +239,7 @@ func TestContextCleanup(t *testing.T) {
// Create the master and the scheduler with the test plugin set. // Create the master and the scheduler with the test plugin set.
context := initTestSchedulerWithOptions(t, context := initTestSchedulerWithOptions(t,
initTestMaster(t, "plugin-context-cleanup", nil), initTestMaster(t, "plugin-context-cleanup", nil),
false, nil, testPluginSet, false, true, time.Second) false, nil, testPluginSet, false, time.Second)
defer cleanupTest(t, context) defer cleanupTest(t, context)
cs := context.clientSet cs := context.clientSet
......
...@@ -33,8 +33,6 @@ import ( ...@@ -33,8 +33,6 @@ import (
"k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
"k8s.io/client-go/informers" "k8s.io/client-go/informers"
coreinformers "k8s.io/client-go/informers/core/v1" coreinformers "k8s.io/client-go/informers/core/v1"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
...@@ -46,7 +44,6 @@ import ( ...@@ -46,7 +44,6 @@ import (
podutil "k8s.io/kubernetes/pkg/api/v1/pod" podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controller/disruption" "k8s.io/kubernetes/pkg/controller/disruption"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler" "k8s.io/kubernetes/pkg/scheduler"
_ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider"
schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
...@@ -91,7 +88,6 @@ func createConfiguratorWithPodInformer( ...@@ -91,7 +88,6 @@ func createConfiguratorWithPodInformer(
PdbInformer: informerFactory.Policy().V1beta1().PodDisruptionBudgets(), PdbInformer: informerFactory.Policy().V1beta1().PodDisruptionBudgets(),
StorageClassInformer: informerFactory.Storage().V1().StorageClasses(), StorageClassInformer: informerFactory.Storage().V1().StorageClasses(),
HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight, HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight,
EnableEquivalenceClassCache: utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache),
DisablePreemption: false, DisablePreemption: false,
PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore,
BindTimeoutSeconds: 600, BindTimeoutSeconds: 600,
...@@ -149,7 +145,7 @@ func initTestScheduler( ...@@ -149,7 +145,7 @@ func initTestScheduler(
) *TestContext { ) *TestContext {
// Pod preemption is enabled by default scheduler configuration, but preemption only happens when PodPriority // Pod preemption is enabled by default scheduler configuration, but preemption only happens when PodPriority
// feature gate is enabled at the same time. // feature gate is enabled at the same time.
return initTestSchedulerWithOptions(t, context, setPodInformer, policy, nil, false, true, time.Second) return initTestSchedulerWithOptions(t, context, setPodInformer, policy, nil, false, time.Second)
} }
// initTestSchedulerWithOptions initializes a test environment and creates a scheduler with default // initTestSchedulerWithOptions initializes a test environment and creates a scheduler with default
...@@ -161,16 +157,8 @@ func initTestSchedulerWithOptions( ...@@ -161,16 +157,8 @@ func initTestSchedulerWithOptions(
policy *schedulerapi.Policy, policy *schedulerapi.Policy,
pluginSet plugins.PluginSet, pluginSet plugins.PluginSet,
disablePreemption bool, disablePreemption bool,
disableEquivalenceCache bool,
resyncPeriod time.Duration, resyncPeriod time.Duration,
) *TestContext { ) *TestContext {
if !disableEquivalenceCache {
defer utilfeaturetesting.SetFeatureGateDuringTest(
t,
utilfeature.DefaultFeatureGate,
features.EnableEquivalenceClassCache, true)()
}
// 1. Create scheduler // 1. Create scheduler
context.informerFactory = informers.NewSharedInformerFactory(context.clientSet, resyncPeriod) context.informerFactory = informers.NewSharedInformerFactory(context.clientSet, resyncPeriod)
...@@ -264,7 +252,7 @@ func initTest(t *testing.T, nsPrefix string) *TestContext { ...@@ -264,7 +252,7 @@ func initTest(t *testing.T, nsPrefix string) *TestContext {
// configuration but with pod preemption disabled. // configuration but with pod preemption disabled.
func initTestDisablePreemption(t *testing.T, nsPrefix string) *TestContext { func initTestDisablePreemption(t *testing.T, nsPrefix string) *TestContext {
return initTestSchedulerWithOptions( return initTestSchedulerWithOptions(
t, initTestMaster(t, nsPrefix, nil), true, nil, nil, true, true, time.Second) t, initTestMaster(t, nsPrefix, nil), true, nil, nil, true, time.Second)
} }
// cleanupTest deletes the scheduler and the test namespace. It should be called // cleanupTest deletes the scheduler and the test namespace. It should be called
......
...@@ -99,7 +99,7 @@ type testPVC struct { ...@@ -99,7 +99,7 @@ type testPVC struct {
func TestVolumeBinding(t *testing.T) { func TestVolumeBinding(t *testing.T) {
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)()
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, true)() defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, true)()
config := setupCluster(t, "volume-scheduling-", 2, 0, 0, true) config := setupCluster(t, "volume-scheduling-", 2, 0, 0)
defer config.teardown() defer config.teardown()
cases := map[string]struct { cases := map[string]struct {
...@@ -270,7 +270,7 @@ func TestVolumeBinding(t *testing.T) { ...@@ -270,7 +270,7 @@ func TestVolumeBinding(t *testing.T) {
func TestVolumeBindingRescheduling(t *testing.T) { func TestVolumeBindingRescheduling(t *testing.T) {
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)()
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, true)() defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, true)()
config := setupCluster(t, "volume-scheduling-", 2, 0, 0, true) config := setupCluster(t, "volume-scheduling-", 2, 0, 0)
defer config.teardown() defer config.teardown()
storageClassName := "local-storage" storageClassName := "local-storage"
...@@ -414,7 +414,7 @@ func TestVolumeBindingDynamicStressSlow(t *testing.T) { ...@@ -414,7 +414,7 @@ func TestVolumeBindingDynamicStressSlow(t *testing.T) {
func testVolumeBindingStress(t *testing.T, schedulerResyncPeriod time.Duration, dynamic bool, provisionDelaySeconds int) { func testVolumeBindingStress(t *testing.T, schedulerResyncPeriod time.Duration, dynamic bool, provisionDelaySeconds int) {
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)()
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, true)() defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, true)()
config := setupCluster(t, "volume-binding-stress-", 1, schedulerResyncPeriod, provisionDelaySeconds, true) config := setupCluster(t, "volume-binding-stress-", 1, schedulerResyncPeriod, provisionDelaySeconds)
defer config.teardown() defer config.teardown()
// Set max volume limit to the number of PVCs the test will create // Set max volume limit to the number of PVCs the test will create
...@@ -504,8 +504,7 @@ func testVolumeBindingStress(t *testing.T, schedulerResyncPeriod time.Duration, ...@@ -504,8 +504,7 @@ func testVolumeBindingStress(t *testing.T, schedulerResyncPeriod time.Duration,
func testVolumeBindingWithAffinity(t *testing.T, anti bool, numNodes, numPods, numPVsFirstNode int) { func testVolumeBindingWithAffinity(t *testing.T, anti bool, numNodes, numPods, numPVsFirstNode int) {
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)()
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, true)() defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, true)()
// TODO: disable equivalence cache until kubernetes/kubernetes#67680 is fixed config := setupCluster(t, "volume-pod-affinity-", numNodes, 0, 0)
config := setupCluster(t, "volume-pod-affinity-", numNodes, 0, 0, true)
defer config.teardown() defer config.teardown()
pods := []*v1.Pod{} pods := []*v1.Pod{}
...@@ -632,7 +631,7 @@ func TestVolumeBindingWithAffinity(t *testing.T) { ...@@ -632,7 +631,7 @@ func TestVolumeBindingWithAffinity(t *testing.T) {
func TestPVAffinityConflict(t *testing.T) { func TestPVAffinityConflict(t *testing.T) {
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)()
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, true)() defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, true)()
config := setupCluster(t, "volume-scheduling-", 3, 0, 0, true) config := setupCluster(t, "volume-scheduling-", 3, 0, 0)
defer config.teardown() defer config.teardown()
pv := makePV("local-pv", classImmediate, "", "", node1) pv := makePV("local-pv", classImmediate, "", "", node1)
...@@ -693,7 +692,7 @@ func TestPVAffinityConflict(t *testing.T) { ...@@ -693,7 +692,7 @@ func TestPVAffinityConflict(t *testing.T) {
func TestVolumeProvision(t *testing.T) { func TestVolumeProvision(t *testing.T) {
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)() defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeScheduling, true)()
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, true)() defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PersistentLocalVolumes, true)()
config := setupCluster(t, "volume-scheduling", 1, 0, 0, true) config := setupCluster(t, "volume-scheduling", 1, 0, 0)
defer config.teardown() defer config.teardown()
cases := map[string]struct { cases := map[string]struct {
...@@ -889,9 +888,8 @@ func TestRescheduleProvisioning(t *testing.T) { ...@@ -889,9 +888,8 @@ func TestRescheduleProvisioning(t *testing.T) {
} }
} }
func setupCluster(t *testing.T, nsName string, numberOfNodes int, resyncPeriod time.Duration, provisionDelaySeconds int, disableEquivalenceCache bool) *testConfig { func setupCluster(t *testing.T, nsName string, numberOfNodes int, resyncPeriod time.Duration, provisionDelaySeconds int) *testConfig {
context := initTestSchedulerWithOptions(t, initTestMaster(t, nsName, nil), false, nil, nil, false, disableEquivalenceCache, resyncPeriod) context := initTestSchedulerWithOptions(t, initTestMaster(t, nsName, nil), false, nil, nil, false, resyncPeriod)
clientset := context.clientSet clientset := context.clientSet
ns := context.ns.Name ns := context.ns.Name
......
...@@ -16,12 +16,10 @@ go_library( ...@@ -16,12 +16,10 @@ go_library(
"//pkg/api/legacyscheme:go_default_library", "//pkg/api/legacyscheme:go_default_library",
"//pkg/cloudprovider/providers/gce:go_default_library", "//pkg/cloudprovider/providers/gce:go_default_library",
"//pkg/cloudprovider/providers/gce/cloud:go_default_library", "//pkg/cloudprovider/providers/gce/cloud:go_default_library",
"//pkg/features:go_default_library",
"//pkg/scheduler:go_default_library", "//pkg/scheduler:go_default_library",
"//pkg/scheduler/api:go_default_library", "//pkg/scheduler/api:go_default_library",
"//pkg/scheduler/factory:go_default_library", "//pkg/scheduler/factory:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
......
...@@ -21,14 +21,12 @@ import ( ...@@ -21,14 +21,12 @@ import (
"net/http/httptest" "net/http/httptest"
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/informers" "k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
clientv1core "k8s.io/client-go/kubernetes/typed/core/v1" clientv1core "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
"k8s.io/klog" "k8s.io/klog"
"k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler" "k8s.io/kubernetes/pkg/scheduler"
schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
"k8s.io/kubernetes/pkg/scheduler/factory" "k8s.io/kubernetes/pkg/scheduler/factory"
...@@ -94,6 +92,7 @@ func createSchedulerConfigurator( ...@@ -94,6 +92,7 @@ func createSchedulerConfigurator(
informerFactory informers.SharedInformerFactory, informerFactory informers.SharedInformerFactory,
stopCh <-chan struct{}, stopCh <-chan struct{},
) factory.Configurator { ) factory.Configurator {
return factory.NewConfigFactory(&factory.ConfigFactoryArgs{ return factory.NewConfigFactory(&factory.ConfigFactoryArgs{
SchedulerName: v1.DefaultSchedulerName, SchedulerName: v1.DefaultSchedulerName,
Client: clientSet, Client: clientSet,
...@@ -108,7 +107,6 @@ func createSchedulerConfigurator( ...@@ -108,7 +107,6 @@ func createSchedulerConfigurator(
PdbInformer: informerFactory.Policy().V1beta1().PodDisruptionBudgets(), PdbInformer: informerFactory.Policy().V1beta1().PodDisruptionBudgets(),
StorageClassInformer: informerFactory.Storage().V1().StorageClasses(), StorageClassInformer: informerFactory.Storage().V1().StorageClasses(),
HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight, HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight,
EnableEquivalenceClassCache: utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache),
DisablePreemption: false, DisablePreemption: false,
PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore,
StopCh: stopCh, StopCh: stopCh,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment