change sort function of scheduling queue to avoid starvation when unschedulable…

pkg/scheduler/internal/queue/scheduling_queue.go

@@ -226,7 +226,10 @@ func podTimestamp(pod *v1.Pod) *metav1.Time {
 	if condition == nil {
 		return &pod.CreationTimestamp
 	}
+	if condition.LastProbeTime.IsZero() {
 		return &condition.LastTransitionTime
+	}
+	return &condition.LastProbeTime
 }
 
 // activeQComp is the function used by the activeQ heap algorithm to sort pods.

pkg/scheduler/internal/queue/scheduling_queue_test.go

@@ -640,6 +640,7 @@ func TestRecentlyTriedPodsGoBack(t *testing.T) {
 		Status:        v1.ConditionFalse,
 		Reason:        v1.PodReasonUnschedulable,
 		Message:       "fake scheduling failure",
+		LastProbeTime: metav1.Now(),
 	})
 	// Put in the unschedulable queue.
 	q.AddUnschedulableIfNotPresent(p1)
@@ -657,3 +658,91 @@ func TestRecentlyTriedPodsGoBack(t *testing.T) {
 		}
 	}
 }
+
+// TestPodFailedSchedulingMultipleTimesDoesNotBlockNewerPod tests
+// that a pod determined as unschedulable multiple times doesn't block any newer pod.
+// This behavior ensures that an unschedulable pod does not block head of the queue when there
+// are frequent events that move pods to the active queue.
+func TestPodFailedSchedulingMultipleTimesDoesNotBlockNewerPod(t *testing.T) {
+	q := NewPriorityQueue()
+
+	// Add an unschedulable pod to a priority queue.
+	// This makes a situation that the pod was tried to schedule
+	// and had been determined unschedulable so far.
+	unschedulablePod := v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pod-unscheduled",
+			Namespace: "ns1",
+			UID:       "tp001",
+		},
+		Spec: v1.PodSpec{
+			Priority: &highPriority,
+		},
+		Status: v1.PodStatus{
+			NominatedNodeName: "node1",
+		},
+	}
+	// Update pod condition to unschedulable.
+	podutil.UpdatePodCondition(&unschedulablePod.Status, &v1.PodCondition{
+		Type:          v1.PodScheduled,
+		Status:        v1.ConditionFalse,
+		Reason:        v1.PodReasonUnschedulable,
+		Message:       "fake scheduling failure",
+		LastProbeTime: metav1.Now(),
+	})
+	// Put in the unschedulable queue
+	q.AddUnschedulableIfNotPresent(&unschedulablePod)
+	// Move all unschedulable pods to the active queue.
+	q.MoveAllToActiveQueue()
+
+	// Simulate a pod being popped by the scheduler,
+	// At this time, unschedulable pod should be popped.
+	p1, err := q.Pop()
+	if err != nil {
+		t.Errorf("Error while popping the head of the queue: %v", err)
+	}
+	if p1 != &unschedulablePod {
+		t.Errorf("Expected that test-pod-unscheduled was popped, got %v", p1.Name)
+	}
+
+	// Assume newer pod was added just after unschedulable pod
+	// being popped and before being pushed back to the queue.
+	newerPod := v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:              "test-newer-pod",
+			Namespace:         "ns1",
+			UID:               "tp002",
+			CreationTimestamp: metav1.Now(),
+		},
+		Spec: v1.PodSpec{
+			Priority: &highPriority,
+		},
+		Status: v1.PodStatus{
+			NominatedNodeName: "node1",
+		},
+	}
+	q.Add(&newerPod)
+
+	// And then unschedulablePod was determined as unschedulable AGAIN.
+	podutil.UpdatePodCondition(&unschedulablePod.Status, &v1.PodCondition{
+		Type:          v1.PodScheduled,
+		Status:        v1.ConditionFalse,
+		Reason:        v1.PodReasonUnschedulable,
+		Message:       "fake scheduling failure",
+		LastProbeTime: metav1.Now(),
+	})
+	// And then, put unschedulable pod to the unschedulable queue
+	q.AddUnschedulableIfNotPresent(&unschedulablePod)
+	// Move all unschedulable pods to the active queue.
+	q.MoveAllToActiveQueue()
+
+	// At this time, newerPod should be popped
+	// because it is the oldest tried pod.
+	p2, err2 := q.Pop()
+	if err2 != nil {
+		t.Errorf("Error while popping the head of the queue: %v", err2)
+	}
+	if p2 != &newerPod {
+		t.Errorf("Expected that test-newer-pod was popped, got %v", p2.Name)
+	}
+}

pkg/scheduler/scheduler.go

@@ -296,6 +296,7 @@ func (sched *Scheduler) schedule(pod *v1.Pod) (string, error) {
 		sched.config.PodConditionUpdater.Update(pod, &v1.PodCondition{
 			Type:          v1.PodScheduled,
 			Status:        v1.ConditionFalse,
+			LastProbeTime: metav1.Now(),
 			Reason:        v1.PodReasonUnschedulable,
 			Message:       err.Error(),
 		})
@@ -375,6 +376,7 @@ func (sched *Scheduler) assumeVolumes(assumed *v1.Pod, host string) (allBound bo
 			sched.config.PodConditionUpdater.Update(assumed, &v1.PodCondition{
 				Type:          v1.PodScheduled,
 				Status:        v1.ConditionFalse,
+				LastProbeTime: metav1.Now(),
 				Reason:        "SchedulerError",
 				Message:       err.Error(),
 			})
@@ -419,6 +421,7 @@ func (sched *Scheduler) bindVolumes(assumed *v1.Pod) error {
 		sched.config.PodConditionUpdater.Update(assumed, &v1.PodCondition{
 			Type:          v1.PodScheduled,
 			Status:        v1.ConditionFalse,
+			LastProbeTime: metav1.Now(),
 			Reason:        reason,
 		})
 		return err
@@ -454,6 +457,7 @@ func (sched *Scheduler) assume(assumed *v1.Pod, host string) error {
 		sched.config.PodConditionUpdater.Update(assumed, &v1.PodCondition{
 			Type:          v1.PodScheduled,
 			Status:        v1.ConditionFalse,
+			LastProbeTime: metav1.Now(),
 			Reason:        "SchedulerError",
 			Message:       err.Error(),
 		})
@@ -493,6 +497,7 @@ func (sched *Scheduler) bind(assumed *v1.Pod, b *v1.Binding) error {
 		sched.config.PodConditionUpdater.Update(assumed, &v1.PodCondition{
 			Type:          v1.PodScheduled,
 			Status:        v1.ConditionFalse,
+			LastProbeTime: metav1.Now(),
 			Reason:        "BindingRejected",
 		})
 		return err