Replace scale down forbidden window

api/api-rules/violation_exceptions.list

@@ -90,6 +90,7 @@ API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alp
 API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,GroupResource,Resource
 API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerSyncPeriod
 API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerUpscaleForbiddenWindow
+API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerDownscaleStabilizationWindow
 API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerDownscaleForbiddenWindow
 API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerTolerance
 API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerUseRESTClients

cmd/kube-controller-manager/app/autoscaling.go

@@ -95,7 +95,7 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
 		replicaCalc,
 		ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
 		ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod.Duration,
-		ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
+		ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration,
 	).Run(ctx.Stop)
 	return nil, true, nil
 }

cmd/kube-controller-manager/app/options/hpacontroller.go

@@ -25,13 +25,14 @@ import (
 
 // HPAControllerOptions holds the HPAController options.
 type HPAControllerOptions struct {
-	HorizontalPodAutoscalerUseRESTClients           bool
-	HorizontalPodAutoscalerTolerance                float64
-	HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
-	HorizontalPodAutoscalerUpscaleForbiddenWindow   metav1.Duration
-	HorizontalPodAutoscalerSyncPeriod               metav1.Duration
-	HorizontalPodAutoscalerCPUInitializationPeriod  metav1.Duration
-	HorizontalPodAutoscalerInitialReadinessDelay    metav1.Duration
+	HorizontalPodAutoscalerUseRESTClients               bool
+	HorizontalPodAutoscalerTolerance                    float64
+	HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
+	HorizontalPodAutoscalerDownscaleForbiddenWindow     metav1.Duration
+	HorizontalPodAutoscalerUpscaleForbiddenWindow       metav1.Duration
+	HorizontalPodAutoscalerSyncPeriod                   metav1.Duration
+	HorizontalPodAutoscalerCPUInitializationPeriod      metav1.Duration
+	HorizontalPodAutoscalerInitialReadinessDelay        metav1.Duration
 }
 
 // AddFlags adds flags related to HPAController for controller manager to the specified FlagSet.
@@ -43,7 +44,9 @@ func (o *HPAControllerOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&o.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", o.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")
 	fs.DurationVar(&o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.")
 	fs.MarkDeprecated("horizontal-pod-autoscaler-upscale-delay", "This flag is currently no-op and will be deleted.")
+	fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, "horizontal-pod-autoscaler-downscale-stabilization", o.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, "The period for which autoscaler will look backwards and not scale down below any recommendation it made during that period.")
 	fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.")
+	fs.MarkDeprecated("horizontal-pod-autoscaler-downscale-delay", "This flag is currently no-op and will be deleted.")
 	fs.Float64Var(&o.HorizontalPodAutoscalerTolerance, "horizontal-pod-autoscaler-tolerance", o.HorizontalPodAutoscalerTolerance, "The minimum change (from 1.0) in the desired-to-actual metrics ratio for the horizontal pod autoscaler to consider scaling.")
 	fs.BoolVar(&o.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", o.HorizontalPodAutoscalerUseRESTClients, "If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy.  This is required for custom metrics support in the horizontal pod autoscaler.")
 	fs.DurationVar(&o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "horizontal-pod-autoscaler-cpu-initialization-period", o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "The period after pod start when CPU samples might be skipped.")
@@ -57,7 +60,7 @@ func (o *HPAControllerOptions) ApplyTo(cfg *componentconfig.HPAControllerConfigu
 	}
 
 	cfg.HorizontalPodAutoscalerSyncPeriod = o.HorizontalPodAutoscalerSyncPeriod
-	cfg.HorizontalPodAutoscalerDownscaleForbiddenWindow = o.HorizontalPodAutoscalerDownscaleForbiddenWindow
+	cfg.HorizontalPodAutoscalerDownscaleStabilizationWindow = o.HorizontalPodAutoscalerDownscaleStabilizationWindow
 	cfg.HorizontalPodAutoscalerTolerance = o.HorizontalPodAutoscalerTolerance
 	cfg.HorizontalPodAutoscalerUseRESTClients = o.HorizontalPodAutoscalerUseRESTClients
 	cfg.HorizontalPodAutoscalerCPUInitializationPeriod = o.HorizontalPodAutoscalerCPUInitializationPeriod

cmd/kube-controller-manager/app/options/options.go

@@ -132,13 +132,14 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) {
 			EnableGarbageCollector: componentConfig.GarbageCollectorController.EnableGarbageCollector,
 		},
 		HPAController: &HPAControllerOptions{
-			HorizontalPodAutoscalerSyncPeriod:               componentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod,
-			HorizontalPodAutoscalerUpscaleForbiddenWindow:   componentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow,
-			HorizontalPodAutoscalerDownscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow,
-			HorizontalPodAutoscalerCPUInitializationPeriod:  componentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod,
-			HorizontalPodAutoscalerInitialReadinessDelay:    componentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay,
-			HorizontalPodAutoscalerTolerance:                componentConfig.HPAController.HorizontalPodAutoscalerTolerance,
-			HorizontalPodAutoscalerUseRESTClients:           componentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients,
+			HorizontalPodAutoscalerSyncPeriod:                   componentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod,
+			HorizontalPodAutoscalerUpscaleForbiddenWindow:       componentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow,
+			HorizontalPodAutoscalerDownscaleForbiddenWindow:     componentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow,
+			HorizontalPodAutoscalerDownscaleStabilizationWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow,
+			HorizontalPodAutoscalerCPUInitializationPeriod:      componentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod,
+			HorizontalPodAutoscalerInitialReadinessDelay:        componentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay,
+			HorizontalPodAutoscalerTolerance:                    componentConfig.HPAController.HorizontalPodAutoscalerTolerance,
+			HorizontalPodAutoscalerUseRESTClients:               componentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients,
 		},
 		JobController: &JobControllerOptions{
 			ConcurrentJobSyncs: componentConfig.JobController.ConcurrentJobSyncs,

cmd/kube-controller-manager/app/options/options_test.go

@@ -75,6 +75,7 @@ func TestAddFlags(t *testing.T) {
 		"--horizontal-pod-autoscaler-downscale-delay=2m",
 		"--horizontal-pod-autoscaler-sync-period=45s",
 		"--horizontal-pod-autoscaler-upscale-delay=1m",
+		"--horizontal-pod-autoscaler-downscale-stabilization=3m",
 		"--horizontal-pod-autoscaler-cpu-initialization-period=90s",
 		"--horizontal-pod-autoscaler-initial-readiness-delay=50s",
 		"--http2-max-streams-per-connection=47",
@@ -186,13 +187,14 @@ func TestAddFlags(t *testing.T) {
 			EnableGarbageCollector: false,
 		},
 		HPAController: &HPAControllerOptions{
-			HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 45 * time.Second},
-			HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 1 * time.Minute},
-			HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute},
-			HorizontalPodAutoscalerCPUInitializationPeriod:  metav1.Duration{Duration: 90 * time.Second},
-			HorizontalPodAutoscalerInitialReadinessDelay:    metav1.Duration{Duration: 50 * time.Second},
-			HorizontalPodAutoscalerTolerance:                0.1,
-			HorizontalPodAutoscalerUseRESTClients:           true,
+			HorizontalPodAutoscalerSyncPeriod:                   metav1.Duration{Duration: 45 * time.Second},
+			HorizontalPodAutoscalerUpscaleForbiddenWindow:       metav1.Duration{Duration: 1 * time.Minute},
+			HorizontalPodAutoscalerDownscaleForbiddenWindow:     metav1.Duration{Duration: 2 * time.Minute},
+			HorizontalPodAutoscalerDownscaleStabilizationWindow: metav1.Duration{Duration: 3 * time.Minute},
+			HorizontalPodAutoscalerCPUInitializationPeriod:      metav1.Duration{Duration: 90 * time.Second},
+			HorizontalPodAutoscalerInitialReadinessDelay:        metav1.Duration{Duration: 50 * time.Second},
+			HorizontalPodAutoscalerTolerance:                    0.1,
+			HorizontalPodAutoscalerUseRESTClients:               true,
 		},
 		JobController: &JobControllerOptions{
 			ConcurrentJobSyncs: 5,

pkg/apis/componentconfig/types.go

@@ -264,6 +264,9 @@ type HPAControllerConfiguration struct {
 	HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
 	// horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
 	HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
+	// HorizontalPodAutoscalerDowncaleStabilizationWindow is a period for which autoscaler will look
+	// backwards and not scale down below any recommendation it made during that period.
+	HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
 	// horizontalPodAutoscalerTolerance is the tolerance for when
 	// resource usage suggests upscaling/downscaling
 	HorizontalPodAutoscalerTolerance float64

pkg/apis/componentconfig/v1alpha1/defaults.go

@@ -89,6 +89,9 @@ func SetDefaults_KubeControllerManagerConfiguration(obj *KubeControllerManagerCo
 	if obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow == zero {
 		obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow = metav1.Duration{Duration: 3 * time.Minute}
 	}
+	if obj.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow == zero {
+		obj.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow = metav1.Duration{Duration: 5 * time.Minute}
+	}
 	if obj.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod == zero {
 		obj.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod = metav1.Duration{Duration: 5 * time.Minute}
 	}

pkg/apis/componentconfig/v1alpha1/types.go

@@ -306,14 +306,17 @@ type GarbageCollectorControllerConfiguration struct {
 }
 
 type HPAControllerConfiguration struct {
-	// horizontalPodAutoscalerSyncPeriod is the period for syncing the number of
+	// HorizontalPodAutoscalerSyncPeriod is the period for syncing the number of
 	// pods in horizontal pod autoscaler.
 	HorizontalPodAutoscalerSyncPeriod metav1.Duration
-	// horizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed.
+	// HorizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed.
 	HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
-	// horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
+	// HorizontalPodAutoscalerDowncaleStabilizationWindow is a period for which autoscaler will look
+	// backwards and not scale down below any recommendation it made during that period.
+	HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
+	// HorizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
 	HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
-	// horizontalPodAutoscalerTolerance is the tolerance for when
+	// HorizontalPodAutoscalerTolerance is the tolerance for when
 	// resource usage suggests upscaling/downscaling
 	HorizontalPodAutoscalerTolerance float64
 	// HorizontalPodAutoscalerUseRESTClients causes the HPA controller to use REST clients

pkg/controller/podautoscaler/BUILD

@@ -83,6 +83,7 @@ go_test(
         "//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/fake:go_default_library",
         "//staging/src/k8s.io/metrics/pkg/client/custom_metrics/fake:go_default_library",
         "//staging/src/k8s.io/metrics/pkg/client/external_metrics/fake:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/github.com/stretchr/testify/require:go_default_library",
         "//vendor/k8s.io/heapster/metrics/api/v1/types:go_default_library",

pkg/controller/podautoscaler/horizontal.go

@@ -53,6 +53,11 @@ var (
 	scaleUpLimitMinimum = 4.0
 )
 
+type timestampedRecommendation struct {
+	recommendation int32
+	timestamp      time.Time
+}
+
 // HorizontalController is responsible for the synchronizing HPA objects stored
 // in the system with the actual deployments/replication controllers they
 // control.
@@ -64,7 +69,7 @@ type HorizontalController struct {
 	replicaCalc   *ReplicaCalculator
 	eventRecorder record.EventRecorder
 
-	downscaleForbiddenWindow time.Duration
+	downscaleStabilisationWindow time.Duration
 
 	// hpaLister is able to list/get HPAs from the shared cache from the informer passed in to
 	// NewHorizontalController.
@@ -73,6 +78,9 @@ type HorizontalController struct {
 
 	// Controllers that need to be synced
 	queue workqueue.RateLimitingInterface
+
+	// Latest unstabilized recommendations for each autoscaler.
+	recommendations map[string][]timestampedRecommendation
 }
 
 // NewHorizontalController creates a new HorizontalController.
@@ -84,7 +92,7 @@ func NewHorizontalController(
 	replicaCalc *ReplicaCalculator,
 	hpaInformer autoscalinginformers.HorizontalPodAutoscalerInformer,
 	resyncPeriod time.Duration,
-	downscaleForbiddenWindow time.Duration,
+	downscaleStabilisationWindow time.Duration,
 
 ) *HorizontalController {
 	broadcaster := record.NewBroadcaster()
@@ -93,13 +101,14 @@ func NewHorizontalController(
 	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "horizontal-pod-autoscaler"})
 
 	hpaController := &HorizontalController{
-		replicaCalc:              replicaCalc,
-		eventRecorder:            recorder,
-		scaleNamespacer:          scaleNamespacer,
-		hpaNamespacer:            hpaNamespacer,
-		downscaleForbiddenWindow: downscaleForbiddenWindow,
-		queue:  workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
-		mapper: mapper,
+		replicaCalc:                  replicaCalc,
+		eventRecorder:                recorder,
+		scaleNamespacer:              scaleNamespacer,
+		hpaNamespacer:                hpaNamespacer,
+		downscaleStabilisationWindow: downscaleStabilisationWindow,
+		queue:           workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
+		mapper:          mapper,
+		recommendations: map[string][]timestampedRecommendation{},
 	}
 
 	hpaInformer.Informer().AddEventHandlerWithResyncPeriod(
@@ -275,10 +284,11 @@ func (a *HorizontalController) reconcileKey(key string) error {
 	hpa, err := a.hpaLister.HorizontalPodAutoscalers(namespace).Get(name)
 	if errors.IsNotFound(err) {
 		glog.Infof("Horizontal Pod Autoscaler %s has been deleted in %s", name, namespace)
+		delete(a.recommendations, key)
 		return nil
 	}
 
-	return a.reconcileAutoscaler(hpa)
+	return a.reconcileAutoscaler(hpa, key)
 }
 
 // computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType.
@@ -431,7 +441,7 @@ func (a *HorizontalController) computeStatusForExternalMetric(currentReplicas in
 	return 0, time.Time{}, "", fmt.Errorf(errMsg)
 }
 
-func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler) error {
+func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler, key string) error {
 	// make a copy so that we never mutate the shared informer cache (conversion can mutate the object)
 	hpav1 := hpav1Shared.DeepCopy()
 	// then, convert to autoscaling/v2, which makes our lives easier when calculating metrics
@@ -527,24 +537,8 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
 		if desiredReplicas < currentReplicas {
 			rescaleReason = "All metrics below target"
 		}
-
-		desiredReplicas = a.normalizeDesiredReplicas(hpa, currentReplicas, desiredReplicas)
-
-		rescale = a.shouldScale(hpa, currentReplicas, desiredReplicas, timestamp)
-		backoffDown := false
-		backoffUp := false
-		if hpa.Status.LastScaleTime != nil {
-			if !hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) {
-				setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionFalse, "BackoffDownscale", "the time since the previous scale is still within the downscale forbidden window")
-				backoffDown = true
-			}
-		}
-
-		if !backoffDown && !backoffUp {
-			// mark that we're not backing off
-			setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "the last scale time was sufficiently old as to warrant a new scale")
-		}
-
+		desiredReplicas = a.normalizeDesiredReplicas(hpa, key, currentReplicas, desiredReplicas)
+		rescale = desiredReplicas != currentReplicas
 	}
 
 	if rescale {
@@ -572,9 +566,39 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
 	return a.updateStatusIfNeeded(hpaStatusOriginal, hpa)
 }
 
+// stabilizeRecommendation:
+// - replaces old recommendation with the newest recommendation,
+// - returns max of recommendations that are not older than downscaleStabilisationWindow.
+func (a *HorizontalController) stabilizeRecommendation(key string, prenormalizedDesiredReplicas int32) int32 {
+	maxRecommendation := prenormalizedDesiredReplicas
+	foundOldSample := false
+	oldSampleIndex := 0
+	cutoff := time.Now().Add(-a.downscaleStabilisationWindow)
+	for i, rec := range a.recommendations[key] {
+		if rec.timestamp.Before(cutoff) {
+			foundOldSample = true
+			oldSampleIndex = i
+		} else if rec.recommendation > maxRecommendation {
+			maxRecommendation = rec.recommendation
+		}
+	}
+	if foundOldSample {
+		a.recommendations[key][oldSampleIndex] = timestampedRecommendation{prenormalizedDesiredReplicas, time.Now()}
+	} else {
+		a.recommendations[key] = append(a.recommendations[key], timestampedRecommendation{prenormalizedDesiredReplicas, time.Now()})
+	}
+	return maxRecommendation
+}
+
 // normalizeDesiredReplicas takes the metrics desired replicas value and normalizes it based on the appropriate conditions (i.e. < maxReplicas, >
 // minReplicas, etc...)
-func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas int32, prenormalizedDesiredReplicas int32) int32 {
+func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, key string, currentReplicas int32, prenormalizedDesiredReplicas int32) int32 {
+	stabilizedRecommendation := a.stabilizeRecommendation(key, prenormalizedDesiredReplicas)
+	if stabilizedRecommendation != prenormalizedDesiredReplicas {
+		setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ScaleDownStabilized", "recent recommendations were higher than current one, applying the highest recent recommendation")
+	} else {
+		setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "recommended size matches current size")
+	}
 	var minReplicas int32
 	if hpa.Spec.MinReplicas != nil {
 		minReplicas = *hpa.Spec.MinReplicas
@@ -582,9 +606,9 @@ func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.Horiz
 		minReplicas = 0
 	}
 
-	desiredReplicas, condition, reason := convertDesiredReplicasWithRules(currentReplicas, prenormalizedDesiredReplicas, minReplicas, hpa.Spec.MaxReplicas)
+	desiredReplicas, condition, reason := convertDesiredReplicasWithRules(currentReplicas, stabilizedRecommendation, minReplicas, hpa.Spec.MaxReplicas)
 
-	if desiredReplicas == prenormalizedDesiredReplicas {
+	if desiredReplicas == stabilizedRecommendation {
 		setCondition(hpa, autoscalingv2.ScalingLimited, v1.ConditionFalse, condition, reason)
 	} else {
 		setCondition(hpa, autoscalingv2.ScalingLimited, v1.ConditionTrue, condition, reason)
@@ -641,29 +665,6 @@ func calculateScaleUpLimit(currentReplicas int32) int32 {
 	return int32(math.Max(scaleUpLimitFactor*float64(currentReplicas), scaleUpLimitMinimum))
 }
 
-func (a *HorizontalController) shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool {
-	if desiredReplicas == currentReplicas {
-		return false
-	}
-
-	if hpa.Status.LastScaleTime == nil {
-		return true
-	}
-
-	// Going down only if the usageRatio dropped significantly below the target
-	// and there was no rescaling in the last downscaleForbiddenWindow.
-	if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) {
-		return true
-	}
-
-	// Going up only if the usage ratio increased significantly above the target.
-	if desiredReplicas > currentReplicas {
-		return true
-	}
-
-	return false
-}
-
 // scaleForResourceMappings attempts to fetch the scale for the
 // resource with the given name and namespace, trying each RESTMapping
 // in turn until a working one is found.  If none work, the first error

pkg/controller/podautoscaler/horizontal_test.go

@@ -51,6 +51,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/golang/glog"
 	_ "k8s.io/kubernetes/pkg/apis/autoscaling/install"
 	_ "k8s.io/kubernetes/pkg/apis/extensions/install"
 )
@@ -129,6 +130,8 @@ type testCase struct {
 	testCMClient      *cmfake.FakeCustomMetricsClient
 	testEMClient      *emfake.FakeExternalMetricsClient
 	testScaleClient   *scalefake.FakeScaleClient
+
+	recommendations []timestampedRecommendation
 }
 
 // Needs to be called under a lock.
@@ -662,7 +665,7 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform
 	replicaCalc := NewReplicaCalculator(metricsClient, testClient.Core(), defaultTestingTolerance, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus)
 
 	informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
-	defaultDownscaleForbiddenWindow := 5 * time.Minute
+	defaultDownscalestabilizationWindow := 5 * time.Minute
 
 	hpaController := NewHorizontalController(
 		eventClient.Core(),
@@ -672,9 +675,12 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform
 		replicaCalc,
 		informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
 		controller.NoResyncPeriodFunc(),
-		defaultDownscaleForbiddenWindow,
+		defaultDownscalestabilizationWindow,
 	)
 	hpaController.hpaListerSynced = alwaysReady
+	if tc.recommendations != nil {
+		hpaController.recommendations["test-namespace/test-hpa"] = tc.recommendations
+	}
 
 	return hpaController, informerFactory
 }
@@ -709,6 +715,7 @@ func (tc *testCase) runTestWithController(t *testing.T, hpaController *Horizonta
 func (tc *testCase) runTest(t *testing.T) {
 	hpaController, informerFactory := tc.setupController(t)
 	tc.runTestWithController(t, hpaController, informerFactory)
+	glog.Errorf("recommendations: %+v", hpaController.recommendations)
 }
 
 func TestScaleUp(t *testing.T) {
@@ -2080,8 +2087,7 @@ func TestNoBackoffUpscaleCMNoBackoffCpu(t *testing.T) {
 	tc.runTest(t)
 }
 
-func TestBackoffDownscale(t *testing.T) {
-	time := metav1.Time{Time: time.Now().Add(-4 * time.Minute)}
+func TestStabilizeDownscale(t *testing.T) {
 	tc := testCase{
 		minReplicas:             1,
 		maxReplicas:             5,
@@ -2091,16 +2097,19 @@ func TestBackoffDownscale(t *testing.T) {
 		reportedLevels:          []uint64{50, 50, 50},
 		reportedCPURequests:     []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
 		useMetricsAPI:           true,
-		lastScaleTime:           &time,
 		expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
 			Type:   autoscalingv2.AbleToScale,
 			Status: v1.ConditionTrue,
 			Reason: "ReadyForNewScale",
 		}, autoscalingv2.HorizontalPodAutoscalerCondition{
 			Type:   autoscalingv2.AbleToScale,
-			Status: v1.ConditionFalse,
-			Reason: "BackoffDownscale",
+			Status: v1.ConditionTrue,
+			Reason: "ScaleDownStabilized",
 		}),
+		recommendations: []timestampedRecommendation{
+			{10, time.Now().Add(-10 * time.Minute)},
+			{4, time.Now().Add(-1 * time.Minute)},
+		},
 	}
 	tc.runTest(t)
 }
@@ -2278,7 +2287,7 @@ func TestAvoidUncessaryUpdates(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	if err := controller.reconcileAutoscaler(&initialHPAs.Items[0]); err != nil {
+	if err := controller.reconcileAutoscaler(&initialHPAs.Items[0], ""); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
 
@@ -2353,4 +2362,85 @@ func TestConvertDesiredReplicasWithRules(t *testing.T) {
 	}
 }
 
+func TestNormalizeDesiredReplicas(t *testing.T) {
+	tests := []struct {
+		name                         string
+		key                          string
+		recommendations              []timestampedRecommendation
+		prenormalizedDesiredReplicas int32
+		expectedStabilizedReplicas   int32
+		expectedLogLength            int
+	}{
+		{
+			"empty log",
+			"",
+			[]timestampedRecommendation{},
+			5,
+			5,
+			1,
+		},
+		{
+			"stabilize",
+			"",
+			[]timestampedRecommendation{
+				{4, time.Now().Add(-2 * time.Minute)},
+				{5, time.Now().Add(-1 * time.Minute)},
+			},
+			3,
+			5,
+			3,
+		},
+		{
+			"no stabilize",
+			"",
+			[]timestampedRecommendation{
+				{1, time.Now().Add(-2 * time.Minute)},
+				{2, time.Now().Add(-1 * time.Minute)},
+			},
+			3,
+			3,
+			3,
+		},
+		{
+			"no stabilize - old recommendations",
+			"",
+			[]timestampedRecommendation{
+				{10, time.Now().Add(-10 * time.Minute)},
+				{9, time.Now().Add(-9 * time.Minute)},
+			},
+			3,
+			3,
+			2,
+		},
+		{
+			"stabilize - old recommendations",
+			"",
+			[]timestampedRecommendation{
+				{10, time.Now().Add(-10 * time.Minute)},
+				{4, time.Now().Add(-1 * time.Minute)},
+				{5, time.Now().Add(-2 * time.Minute)},
+				{9, time.Now().Add(-9 * time.Minute)},
+			},
+			3,
+			5,
+			4,
+		},
+	}
+	for _, tc := range tests {
+		hc := HorizontalController{
+			downscaleStabilisationWindow: 5 * time.Minute,
+			recommendations: map[string][]timestampedRecommendation{
+				tc.key: tc.recommendations,
+			},
+		}
+		r := hc.stabilizeRecommendation(tc.key, tc.prenormalizedDesiredReplicas)
+		if r != tc.expectedStabilizedReplicas {
+			t.Errorf("[%s] got %d stabilized replicas, expected %d", tc.name, r, tc.expectedStabilizedReplicas)
+		}
+		if len(hc.recommendations[tc.key]) != tc.expectedLogLength {
+			t.Errorf("[%s] after  stabilization recommendations log has %d entries, expected %d", tc.name, len(hc.recommendations[tc.key]), tc.expectedLogLength)
+		}
+	}
+}
+
 // TODO: add more tests

pkg/controller/podautoscaler/legacy_horizontal_test.go

@@ -488,7 +488,7 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
 	replicaCalc := NewReplicaCalculator(metricsClient, testClient.Core(), defaultTestingTolerance, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus)
 
 	informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
-	defaultDownscaleForbiddenWindow := 5 * time.Minute
+	defaultDownscaleStabilisationWindow := 5 * time.Minute
 
 	hpaController := NewHorizontalController(
 		eventClient.Core(),
@@ -498,7 +498,7 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
 		replicaCalc,
 		informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
 		controller.NoResyncPeriodFunc(),
-		defaultDownscaleForbiddenWindow,
+		defaultDownscaleStabilisationWindow,
 	)
 	hpaController.hpaListerSynced = alwaysReady