make hpa upscale and downscale delay window configurable

cmd/kube-controller-manager/app/autoscaling.go

@@ -76,6 +76,8 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
 		replicaCalc,
 		ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
 		ctx.Options.HorizontalPodAutoscalerSyncPeriod.Duration,
+		ctx.Options.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration,
+		ctx.Options.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
 	).Run(ctx.Stop)
 	return true, nil
 }

cmd/kube-controller-manager/app/options/options.go

@@ -49,39 +49,41 @@ type CMServer struct {
 func NewCMServer() *CMServer {
 	s := CMServer{
 		KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
-			Controllers:                       []string{"*"},
-			Port:                              ports.ControllerManagerPort,
-			Address:                           "0.0.0.0",
-			ConcurrentEndpointSyncs:           5,
-			ConcurrentServiceSyncs:            1,
-			ConcurrentRCSyncs:                 5,
-			ConcurrentRSSyncs:                 5,
-			ConcurrentDaemonSetSyncs:          2,
-			ConcurrentJobSyncs:                5,
-			ConcurrentResourceQuotaSyncs:      5,
-			ConcurrentDeploymentSyncs:         5,
-			ConcurrentNamespaceSyncs:          2,
-			ConcurrentSATokenSyncs:            5,
-			LookupCacheSizeForRC:              4096,
-			LookupCacheSizeForRS:              4096,
-			LookupCacheSizeForDaemonSet:       1024,
-			ServiceSyncPeriod:                 metav1.Duration{Duration: 5 * time.Minute},
-			RouteReconciliationPeriod:         metav1.Duration{Duration: 10 * time.Second},
-			ResourceQuotaSyncPeriod:           metav1.Duration{Duration: 5 * time.Minute},
-			NamespaceSyncPeriod:               metav1.Duration{Duration: 5 * time.Minute},
-			PVClaimBinderSyncPeriod:           metav1.Duration{Duration: 15 * time.Second},
-			HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
-			DeploymentControllerSyncPeriod:    metav1.Duration{Duration: 30 * time.Second},
-			MinResyncPeriod:                   metav1.Duration{Duration: 12 * time.Hour},
-			RegisterRetryCount:                10,
-			PodEvictionTimeout:                metav1.Duration{Duration: 5 * time.Minute},
-			NodeMonitorGracePeriod:            metav1.Duration{Duration: 40 * time.Second},
-			NodeStartupGracePeriod:            metav1.Duration{Duration: 60 * time.Second},
-			NodeMonitorPeriod:                 metav1.Duration{Duration: 5 * time.Second},
-			ClusterName:                       "kubernetes",
-			NodeCIDRMaskSize:                  24,
-			ConfigureCloudRoutes:              true,
-			TerminatedPodGCThreshold:          12500,
+			Controllers:                                     []string{"*"},
+			Port:                                            ports.ControllerManagerPort,
+			Address:                                         "0.0.0.0",
+			ConcurrentEndpointSyncs:                         5,
+			ConcurrentServiceSyncs:                          1,
+			ConcurrentRCSyncs:                               5,
+			ConcurrentRSSyncs:                               5,
+			ConcurrentDaemonSetSyncs:                        2,
+			ConcurrentJobSyncs:                              5,
+			ConcurrentResourceQuotaSyncs:                    5,
+			ConcurrentDeploymentSyncs:                       5,
+			ConcurrentNamespaceSyncs:                        2,
+			ConcurrentSATokenSyncs:                          5,
+			LookupCacheSizeForRC:                            4096,
+			LookupCacheSizeForRS:                            4096,
+			LookupCacheSizeForDaemonSet:                     1024,
+			ServiceSyncPeriod:                               metav1.Duration{Duration: 5 * time.Minute},
+			RouteReconciliationPeriod:                       metav1.Duration{Duration: 10 * time.Second},
+			ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 5 * time.Minute},
+			NamespaceSyncPeriod:                             metav1.Duration{Duration: 5 * time.Minute},
+			PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 15 * time.Second},
+			HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 30 * time.Second},
+			HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 3 * time.Minute},
+			HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
+			DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 30 * time.Second},
+			MinResyncPeriod:                                 metav1.Duration{Duration: 12 * time.Hour},
+			RegisterRetryCount:                              10,
+			PodEvictionTimeout:                              metav1.Duration{Duration: 5 * time.Minute},
+			NodeMonitorGracePeriod:                          metav1.Duration{Duration: 40 * time.Second},
+			NodeStartupGracePeriod:                          metav1.Duration{Duration: 60 * time.Second},
+			NodeMonitorPeriod:                               metav1.Duration{Duration: 5 * time.Second},
+			ClusterName:                                     "kubernetes",
+			NodeCIDRMaskSize:                                24,
+			ConfigureCloudRoutes:                            true,
+			TerminatedPodGCThreshold:                        12500,
 			VolumeConfiguration: componentconfig.VolumeConfiguration{
 				EnableHostPathProvisioning: false,
 				EnableDynamicProvisioning:  true,
@@ -161,6 +163,8 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled
 	fs.StringVar(&s.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", s.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.")
 	fs.Int32Var(&s.TerminatedPodGCThreshold, "terminated-pod-gc-threshold", s.TerminatedPodGCThreshold, "Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.")
 	fs.DurationVar(&s.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", s.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")
+	fs.DurationVar(&s.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", s.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.")
+	fs.DurationVar(&s.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", s.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.")
 	fs.DurationVar(&s.DeploymentControllerSyncPeriod.Duration, "deployment-controller-sync-period", s.DeploymentControllerSyncPeriod.Duration, "Period for syncing the deployments.")
 	fs.DurationVar(&s.PodEvictionTimeout.Duration, "pod-eviction-timeout", s.PodEvictionTimeout.Duration, "The grace period for deleting pods on failed nodes.")
 	fs.Float32Var(&s.DeletingPodsQps, "deleting-pods-qps", 0.1, "Number of nodes per second on which pods are deleted in case of node failure.")

hack/verify-flags/known-flags.txt

@@ -300,6 +300,8 @@ heapster-port
 heapster-scheme
 heapster-service
 horizontal-pod-autoscaler-sync-period
+horizontal-pod-autoscaler-upscale-delay
+horizontal-pod-autoscaler-downscale-delay
 host-cluster-context
 host-ipc-sources
 hostname-override

pkg/apis/componentconfig/types.go

@@ -725,6 +725,10 @@ type KubeControllerManagerConfiguration struct {
 	// horizontalPodAutoscalerSyncPeriod is the period for syncing the number of
 	// pods in horizontal pod autoscaler.
 	HorizontalPodAutoscalerSyncPeriod metav1.Duration
+	// horizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed.
+	HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
+	// horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
+	HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
 	// deploymentControllerSyncPeriod is the period for syncing the deployments.
 	DeploymentControllerSyncPeriod metav1.Duration
 	// podEvictionTimeout is the grace period for deleting pods on failed nodes.

pkg/controller/podautoscaler/horizontal.go

@@ -85,6 +85,9 @@ type HorizontalController struct {
 	replicaCalc   *ReplicaCalculator
 	eventRecorder record.EventRecorder
 
+	upscaleForbiddenWindow   time.Duration
+	downscaleForbiddenWindow time.Duration
+
 	// hpaLister is able to list/get HPAs from the shared cache from the informer passed in to
 	// NewHorizontalController.
 	hpaLister       autoscalinglisters.HorizontalPodAutoscalerLister
@@ -94,9 +97,6 @@ type HorizontalController struct {
 	queue workqueue.RateLimitingInterface
 }
 
-var downscaleForbiddenWindow = 5 * time.Minute
-var upscaleForbiddenWindow = 3 * time.Minute
-
 func NewHorizontalController(
 	evtNamespacer v1core.EventsGetter,
 	scaleNamespacer extensionsclient.ScalesGetter,
@@ -104,6 +104,9 @@ func NewHorizontalController(
 	replicaCalc *ReplicaCalculator,
 	hpaInformer autoscalinginformers.HorizontalPodAutoscalerInformer,
 	resyncPeriod time.Duration,
+	upscaleForbiddenWindow time.Duration,
+	downscaleForbiddenWindow time.Duration,
+
 ) *HorizontalController {
 	broadcaster := record.NewBroadcaster()
 	// TODO: remove the wrapper when every clients have moved to use the clientset.
@@ -111,11 +114,13 @@ func NewHorizontalController(
 	recorder := broadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "horizontal-pod-autoscaler"})
 
 	hpaController := &HorizontalController{
-		replicaCalc:     replicaCalc,
-		eventRecorder:   recorder,
-		scaleNamespacer: scaleNamespacer,
-		hpaNamespacer:   hpaNamespacer,
-		queue:           workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
+		replicaCalc:              replicaCalc,
+		eventRecorder:            recorder,
+		scaleNamespacer:          scaleNamespacer,
+		hpaNamespacer:            hpaNamespacer,
+		upscaleForbiddenWindow:   upscaleForbiddenWindow,
+		downscaleForbiddenWindow: downscaleForbiddenWindow,
+		queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
 	}
 
 	hpaInformer.Informer().AddEventHandlerWithResyncPeriod(
@@ -434,7 +439,7 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
 			desiredReplicas = calculateScaleUpLimit(currentReplicas)
 		}
 
-		rescale = shouldScale(hpa, currentReplicas, desiredReplicas, timestamp)
+		rescale = a.shouldScale(hpa, currentReplicas, desiredReplicas, timestamp)
 	}
 
 	if rescale {
@@ -455,7 +460,7 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
 	return a.updateStatus(hpa, currentReplicas, desiredReplicas, metricStatuses, rescale)
 }
 
-func shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool {
+func (a *HorizontalController) shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool {
 	if desiredReplicas == currentReplicas {
 		return false
 	}
@@ -466,13 +471,13 @@ func shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, de
 
 	// Going down only if the usageRatio dropped significantly below the target
 	// and there was no rescaling in the last downscaleForbiddenWindow.
-	if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(downscaleForbiddenWindow).Before(timestamp) {
+	if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) {
 		return true
 	}
 
 	// Going up only if the usage ratio increased significantly above the target
 	// and there was no rescaling in the last upscaleForbiddenWindow.
-	if desiredReplicas > currentReplicas && hpa.Status.LastScaleTime.Add(upscaleForbiddenWindow).Before(timestamp) {
+	if desiredReplicas > currentReplicas && hpa.Status.LastScaleTime.Add(a.upscaleForbiddenWindow).Before(timestamp) {
 		return true
 	}
 

pkg/controller/podautoscaler/horizontal_test.go

@@ -518,6 +518,8 @@ func (tc *testCase) runTest(t *testing.T) {
 	}
 
 	informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
+	defaultUpscaleForbiddenWindow := 3 * time.Minute
+	defaultDownscaleForbiddenWindow := 5 * time.Minute
 
 	hpaController := NewHorizontalController(
 		eventClient.Core(),
@@ -526,6 +528,8 @@ func (tc *testCase) runTest(t *testing.T) {
 		replicaCalc,
 		informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
 		controller.NoResyncPeriodFunc(),
+		defaultUpscaleForbiddenWindow,
+		defaultDownscaleForbiddenWindow,
 	)
 	hpaController.hpaListerSynced = alwaysReady
 

pkg/controller/podautoscaler/legacy_horizontal_test.go

@@ -489,6 +489,8 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
 	}
 
 	informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
+	defaultUpscaleForbiddenWindow := 3 * time.Minute
+	defaultDownscaleForbiddenWindow := 5 * time.Minute
 
 	hpaController := NewHorizontalController(
 		eventClient.Core(),
@@ -497,6 +499,8 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
 		replicaCalc,
 		informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
 		controller.NoResyncPeriodFunc(),
+		defaultUpscaleForbiddenWindow,
+		defaultDownscaleForbiddenWindow,
 	)
 	hpaController.hpaListerSynced = alwaysReady