Skip to content

K
k3s
Commit 6e9012c9 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #52137 from aveshagarwal/master-scheduler-resources-consolidation

Automatic merge from submit-queue (batch tested with PRs 51902, 52718, 52687, 52137, 52697). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.. Consolidate extended resources and hugepages in Scheduler Fixes https://github.com/kubernetes/kubernetes/issues/51732 @bsalamat @derekwaynecarr @sjenning @kubernetes/sig-scheduling-pr-reviews
parents a1a441f7 ae05a6da
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 55 additions and 89 deletions
pkg/api/helper/helpers.go
View file @ 6e9012c9
...@@ -265,6 +265,11 @@ func IsIntegerResourceName(str string) bool { ...@@ -265,6 +265,11 @@ func IsIntegerResourceName(str string) bool {
return integerResources.Has(str) || IsExtendedResourceName(api.ResourceName(str)) return integerResources.Has(str) || IsExtendedResourceName(api.ResourceName(str))
} }
// Extended and HugePages resources
func IsScalarResourceName(name api.ResourceName) bool {
return IsExtendedResourceName(name) || IsHugePageResourceName(name)
}
// this function aims to check if the service's ClusterIP is set or not // this function aims to check if the service's ClusterIP is set or not
// the objective is not to perform validation here // the objective is not to perform validation here
func IsServiceIPSet(service *api.Service) bool { func IsServiceIPSet(service *api.Service) bool {
......
pkg/api/v1/helper/helpers.go
View file @ 6e9012c9
...@@ -88,12 +88,18 @@ func OpaqueIntResourceName(name string) v1.ResourceName { ...@@ -88,12 +88,18 @@ func OpaqueIntResourceName(name string) v1.ResourceName {
var overcommitBlacklist = sets.NewString(string(v1.ResourceNvidiaGPU)) var overcommitBlacklist = sets.NewString(string(v1.ResourceNvidiaGPU))
// IsOvercommitAllowed returns true if the resource is in the default // IsOvercommitAllowed returns true if the resource is in the default
// namespace and not blacklisted. // namespace and not blacklisted and is not hugepages.
func IsOvercommitAllowed(name v1.ResourceName) bool { func IsOvercommitAllowed(name v1.ResourceName) bool {
return IsDefaultNamespaceResource(name) && return IsDefaultNamespaceResource(name) &&
!IsHugePageResourceName(name) &&
!overcommitBlacklist.Has(string(name)) !overcommitBlacklist.Has(string(name))
} }
// Extended and Hugepages resources
func IsScalarResourceName(name v1.ResourceName) bool {
return IsExtendedResourceName(name) || IsHugePageResourceName(name)
}
// this function aims to check if the service's ClusterIP is set or not // this function aims to check if the service's ClusterIP is set or not
// the objective is not to perform validation here // the objective is not to perform validation here
func IsServiceIPSet(service *v1.Service) bool { func IsServiceIPSet(service *v1.Service) bool {
......
plugin/pkg/scheduler/algorithm/predicates/predicates.go
View file @ 6e9012c9
...@@ -515,16 +515,10 @@ func GetResourceRequest(pod *v1.Pod) *schedulercache.Resource { ...@@ -515,16 +515,10 @@ func GetResourceRequest(pod *v1.Pod) *schedulercache.Resource {
result.NvidiaGPU = gpu result.NvidiaGPU = gpu
} }
default: default:
if v1helper.IsExtendedResourceName(rName) { if v1helper.IsScalarResourceName(rName) {
value := rQuantity.Value() value := rQuantity.Value()
if value > result.ExtendedResources[rName] { if value > result.ScalarResources[rName] {
result.SetExtended(rName, value) result.SetScalar(rName, value)
}
}
if v1helper.IsHugePageResourceName(rName) {
value := rQuantity.Value()
if value > result.HugePages[rName] {
result.SetHugePages(rName, value)
} }
} }
} }
...@@ -564,8 +558,7 @@ func PodFitsResources(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *s ...@@ -564,8 +558,7 @@ func PodFitsResources(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *s
podRequest.Memory == 0 && podRequest.Memory == 0 &&
podRequest.NvidiaGPU == 0 && podRequest.NvidiaGPU == 0 &&
podRequest.EphemeralStorage == 0 && podRequest.EphemeralStorage == 0 &&
len(podRequest.ExtendedResources) == 0 && len(podRequest.ScalarResources) == 0 {
len(podRequest.HugePages) == 0 {
return len(predicateFails) == 0, predicateFails, nil return len(predicateFails) == 0, predicateFails, nil
} }
...@@ -584,15 +577,9 @@ func PodFitsResources(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *s ...@@ -584,15 +577,9 @@ func PodFitsResources(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *s
predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceEphemeralStorage, podRequest.EphemeralStorage, nodeInfo.RequestedResource().EphemeralStorage, allocatable.EphemeralStorage)) predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceEphemeralStorage, podRequest.EphemeralStorage, nodeInfo.RequestedResource().EphemeralStorage, allocatable.EphemeralStorage))
} }
for rName, rQuant := range podRequest.ExtendedResources { for rName, rQuant := range podRequest.ScalarResources {
if allocatable.ExtendedResources[rName] < rQuant+nodeInfo.RequestedResource().ExtendedResources[rName] { if allocatable.ScalarResources[rName] < rQuant+nodeInfo.RequestedResource().ScalarResources[rName] {
predicateFails = append(predicateFails, NewInsufficientResourceError(rName, podRequest.ExtendedResources[rName], nodeInfo.RequestedResource().ExtendedResources[rName], allocatable.ExtendedResources[rName])) predicateFails = append(predicateFails, NewInsufficientResourceError(rName, podRequest.ScalarResources[rName], nodeInfo.RequestedResource().ScalarResources[rName], allocatable.ScalarResources[rName]))
}
}
for rName, rQuant := range podRequest.HugePages {
if allocatable.HugePages[rName] < rQuant+nodeInfo.RequestedResource().HugePages[rName] {
predicateFails = append(predicateFails, NewInsufficientResourceError(rName, podRequest.HugePages[rName], nodeInfo.RequestedResource().HugePages[rName], allocatable.HugePages[rName]))
} }
} }
......
plugin/pkg/scheduler/schedulercache/cache_test.go
View file @ 6e9012c9
...@@ -109,9 +109,9 @@ func TestAssumePodScheduled(t *testing.T) { ...@@ -109,9 +109,9 @@ func TestAssumePodScheduled(t *testing.T) {
pods: []*v1.Pod{testPods[4]}, pods: []*v1.Pod{testPods[4]},
wNodeInfo: &NodeInfo{ wNodeInfo: &NodeInfo{
requestedResource: &Resource{ requestedResource: &Resource{
MilliCPU: 100, MilliCPU: 100,
Memory: 500, Memory: 500,
ExtendedResources: map[v1.ResourceName]int64{"pod.alpha.kubernetes.io/opaque-int-resource-oir-foo": 3}, ScalarResources: map[v1.ResourceName]int64{"pod.alpha.kubernetes.io/opaque-int-resource-oir-foo": 3},
}, },
nonzeroRequest: &Resource{ nonzeroRequest: &Resource{
MilliCPU: 100, MilliCPU: 100,
...@@ -125,9 +125,9 @@ func TestAssumePodScheduled(t *testing.T) { ...@@ -125,9 +125,9 @@ func TestAssumePodScheduled(t *testing.T) {
pods: []*v1.Pod{testPods[4], testPods[5]}, pods: []*v1.Pod{testPods[4], testPods[5]},
wNodeInfo: &NodeInfo{ wNodeInfo: &NodeInfo{
requestedResource: &Resource{ requestedResource: &Resource{
MilliCPU: 300, MilliCPU: 300,
Memory: 1524, Memory: 1524,
ExtendedResources: map[v1.ResourceName]int64{"pod.alpha.kubernetes.io/opaque-int-resource-oir-foo": 8}, ScalarResources: map[v1.ResourceName]int64{"pod.alpha.kubernetes.io/opaque-int-resource-oir-foo": 8},
}, },
nonzeroRequest: &Resource{ nonzeroRequest: &Resource{
MilliCPU: 300, MilliCPU: 300,
......
plugin/pkg/scheduler/schedulercache/node_info.go
View file @ 6e9012c9
...@@ -70,9 +70,9 @@ type Resource struct { ...@@ -70,9 +70,9 @@ type Resource struct {
EphemeralStorage int64 EphemeralStorage int64
// We store allowedPodNumber (which is Node.Status.Allocatable.Pods().Value()) // We store allowedPodNumber (which is Node.Status.Allocatable.Pods().Value())
// explicitly as int, to avoid conversions and improve performance. // explicitly as int, to avoid conversions and improve performance.
AllowedPodNumber int AllowedPodNumber int
ExtendedResources map[v1.ResourceName]int64 // ScalarResources
HugePages map[v1.ResourceName]int64 ScalarResources map[v1.ResourceName]int64
} }
// New creates a Resource from ResourceList // New creates a Resource from ResourceList
...@@ -101,11 +101,8 @@ func (r *Resource) Add(rl v1.ResourceList) { ...@@ -101,11 +101,8 @@ func (r *Resource) Add(rl v1.ResourceList) {
case v1.ResourceEphemeralStorage: case v1.ResourceEphemeralStorage:
r.EphemeralStorage += rQuant.Value() r.EphemeralStorage += rQuant.Value()
default: default:
if v1helper.IsExtendedResourceName(rName) { if v1helper.IsScalarResourceName(rName) {
r.AddExtended(rName, rQuant.Value()) r.AddScalar(rName, rQuant.Value())
}
if v1helper.IsHugePageResourceName(rName) {
r.AddHugePages(rName, rQuant.Value())
} }
} }
} }
...@@ -119,11 +116,12 @@ func (r *Resource) ResourceList() v1.ResourceList { ...@@ -119,11 +116,12 @@ func (r *Resource) ResourceList() v1.ResourceList {
v1.ResourcePods: *resource.NewQuantity(int64(r.AllowedPodNumber), resource.BinarySI), v1.ResourcePods: *resource.NewQuantity(int64(r.AllowedPodNumber), resource.BinarySI),
v1.ResourceEphemeralStorage: *resource.NewQuantity(r.EphemeralStorage, resource.BinarySI), v1.ResourceEphemeralStorage: *resource.NewQuantity(r.EphemeralStorage, resource.BinarySI),
} }
for rName, rQuant := range r.ExtendedResources { for rName, rQuant := range r.ScalarResources {
result[rName] = *resource.NewQuantity(rQuant, resource.DecimalSI) if v1helper.IsHugePageResourceName(rName) {
} result[rName] = *resource.NewQuantity(rQuant, resource.BinarySI)
for rName, rQuant := range r.HugePages { } else {
result[rName] = *resource.NewQuantity(rQuant, resource.BinarySI) result[rName] = *resource.NewQuantity(rQuant, resource.DecimalSI)
}
} }
return result return result
} }
...@@ -136,43 +134,25 @@ func (r *Resource) Clone() *Resource { ...@@ -136,43 +134,25 @@ func (r *Resource) Clone() *Resource {
AllowedPodNumber: r.AllowedPodNumber, AllowedPodNumber: r.AllowedPodNumber,
EphemeralStorage: r.EphemeralStorage, EphemeralStorage: r.EphemeralStorage,
} }
if r.ExtendedResources != nil { if r.ScalarResources != nil {
res.ExtendedResources = make(map[v1.ResourceName]int64) res.ScalarResources = make(map[v1.ResourceName]int64)
for k, v := range r.ExtendedResources { for k, v := range r.ScalarResources {
res.ExtendedResources[k] = v res.ScalarResources[k] = v
}
}
if r.HugePages != nil {
res.HugePages = make(map[v1.ResourceName]int64)
for k, v := range r.HugePages {
res.HugePages[k] = v
} }
} }
return res return res
} }
func (r *Resource) AddExtended(name v1.ResourceName, quantity int64) { func (r *Resource) AddScalar(name v1.ResourceName, quantity int64) {
r.SetExtended(name, r.ExtendedResources[name]+quantity) r.SetScalar(name, r.ScalarResources[name]+quantity)
} }
func (r *Resource) SetExtended(name v1.ResourceName, quantity int64) { func (r *Resource) SetScalar(name v1.ResourceName, quantity int64) {
// Lazily allocate opaque integer resource map. // Lazily allocate scalar resource map.
if r.ExtendedResources == nil { if r.ScalarResources == nil {
r.ExtendedResources = map[v1.ResourceName]int64{} r.ScalarResources = map[v1.ResourceName]int64{}
} }
r.ExtendedResources[name] = quantity r.ScalarResources[name] = quantity
}
func (r *Resource) AddHugePages(name v1.ResourceName, quantity int64) {
r.SetHugePages(name, r.HugePages[name]+quantity)
}
func (r *Resource) SetHugePages(name v1.ResourceName, quantity int64) {
// Lazily allocate hugepages resource map.
if r.HugePages == nil {
r.HugePages = map[v1.ResourceName]int64{}
}
r.HugePages[name] = quantity
} }
// NewNodeInfo returns a ready to use empty NodeInfo object. // NewNodeInfo returns a ready to use empty NodeInfo object.
...@@ -326,17 +306,11 @@ func (n *NodeInfo) AddPod(pod *v1.Pod) { ...@@ -326,17 +306,11 @@ func (n *NodeInfo) AddPod(pod *v1.Pod) {
n.requestedResource.Memory += res.Memory n.requestedResource.Memory += res.Memory
n.requestedResource.NvidiaGPU += res.NvidiaGPU n.requestedResource.NvidiaGPU += res.NvidiaGPU
n.requestedResource.EphemeralStorage += res.EphemeralStorage n.requestedResource.EphemeralStorage += res.EphemeralStorage
if n.requestedResource.ExtendedResources == nil && len(res.ExtendedResources) > 0 { if n.requestedResource.ScalarResources == nil && len(res.ScalarResources) > 0 {
n.requestedResource.ExtendedResources = map[v1.ResourceName]int64{} n.requestedResource.ScalarResources = map[v1.ResourceName]int64{}
}
for rName, rQuant := range res.ExtendedResources {
n.requestedResource.ExtendedResources[rName] += rQuant
} }
if n.requestedResource.HugePages == nil && len(res.HugePages) > 0 { for rName, rQuant := range res.ScalarResources {
n.requestedResource.HugePages = map[v1.ResourceName]int64{} n.requestedResource.ScalarResources[rName] += rQuant
}
for rName, rQuant := range res.HugePages {
n.requestedResource.HugePages[rName] += rQuant
} }
n.nonzeroRequest.MilliCPU += non0_cpu n.nonzeroRequest.MilliCPU += non0_cpu
n.nonzeroRequest.Memory += non0_mem n.nonzeroRequest.Memory += non0_mem
...@@ -387,17 +361,11 @@ func (n *NodeInfo) RemovePod(pod *v1.Pod) error { ...@@ -387,17 +361,11 @@ func (n *NodeInfo) RemovePod(pod *v1.Pod) error {
n.requestedResource.MilliCPU -= res.MilliCPU n.requestedResource.MilliCPU -= res.MilliCPU
n.requestedResource.Memory -= res.Memory n.requestedResource.Memory -= res.Memory
n.requestedResource.NvidiaGPU -= res.NvidiaGPU n.requestedResource.NvidiaGPU -= res.NvidiaGPU
if len(res.ExtendedResources) > 0 && n.requestedResource.ExtendedResources == nil { if len(res.ScalarResources) > 0 && n.requestedResource.ScalarResources == nil {
n.requestedResource.ExtendedResources = map[v1.ResourceName]int64{} n.requestedResource.ScalarResources = map[v1.ResourceName]int64{}
}
for rName, rQuant := range res.ExtendedResources {
n.requestedResource.ExtendedResources[rName] -= rQuant
}
if len(res.HugePages) > 0 && n.requestedResource.HugePages == nil {
n.requestedResource.HugePages = map[v1.ResourceName]int64{}
} }
for rName, rQuant := range res.HugePages { for rName, rQuant := range res.ScalarResources {
n.requestedResource.HugePages[rName] -= rQuant n.requestedResource.ScalarResources[rName] -= rQuant
} }
n.nonzeroRequest.MilliCPU -= non0_cpu n.nonzeroRequest.MilliCPU -= non0_cpu
n.nonzeroRequest.Memory -= non0_mem n.nonzeroRequest.Memory -= non0_mem
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment