Merge pull request #41272 from DirectXMan12/feature/hpa-v2-controller

pkg/api/testing/fuzzer.go

@@ -547,12 +547,24 @@ func autoscalingFuncs(t apitesting.TestingCommon) []interface{} {
 			minReplicas := int32(c.Rand.Int31())
 			s.MinReplicas = &minReplicas
 
-			// NB: since this is used for round-tripping, we can only fuzz
-			// fields that round-trip successfully, so only the resource source
-			// type is usable here
+			randomQuantity := func() resource.Quantity {
+				var q resource.Quantity
+				c.Fuzz(&q)
+				// precalc the string for benchmarking purposes
+				_ = q.String()
+				return q
+			}
+
 			targetUtilization := int32(c.RandUint64())
 			s.Metrics = []autoscaling.MetricSpec{
 				{
+					Type: autoscaling.PodsMetricSourceType,
+					Pods: &autoscaling.PodsMetricSource{
+						MetricName:         c.RandString(),
+						TargetAverageValue: randomQuantity(),
+					},
+				},
+				{
 					Type: autoscaling.ResourceMetricSourceType,
 					Resource: &autoscaling.ResourceMetricSource{
 						Name: api.ResourceCPU,
@@ -563,12 +575,23 @@ func autoscalingFuncs(t apitesting.TestingCommon) []interface{} {
 		},
 		func(s *autoscaling.HorizontalPodAutoscalerStatus, c fuzz.Continue) {
 			c.FuzzNoCustom(s) // fuzz self without calling this function again
-			// NB: since this is used for round-tripping, we can only fuzz
-			// fields that round-trip successfully, so only the resource status
-			// type is usable here
+			randomQuantity := func() resource.Quantity {
+				var q resource.Quantity
+				c.Fuzz(&q)
+				// precalc the string for benchmarking purposes
+				_ = q.String()
+				return q
+			}
 			currentUtilization := int32(c.RandUint64())
 			s.CurrentMetrics = []autoscaling.MetricStatus{
 				{
+					Type: autoscaling.PodsMetricSourceType,
+					Pods: &autoscaling.PodsMetricStatus{
+						MetricName:          c.RandString(),
+						CurrentAverageValue: randomQuantity(),
+					},
+				},
+				{
 					Type: autoscaling.ResourceMetricSourceType,
 					Resource: &autoscaling.ResourceMetricStatus{
 						Name: api.ResourceCPU,

pkg/apis/autoscaling/v1/conversion.go

@@ -108,12 +108,17 @@ func Convert_v1_HorizontalPodAutoscaler_To_autoscaling_HorizontalPodAutoscaler(i
 			return err
 		}
 
-		out.Spec.Metrics = make([]autoscaling.MetricSpec, len(otherMetrics))
+		// the normal Spec conversion could have populated out.Spec.Metrics with a single element, so deal with that
+		outMetrics := make([]autoscaling.MetricSpec, len(otherMetrics)+len(out.Spec.Metrics))
 		for i, metric := range otherMetrics {
-			if err := Convert_v1_MetricSpec_To_autoscaling_MetricSpec(&metric, &out.Spec.Metrics[i], s); err != nil {
+			if err := Convert_v1_MetricSpec_To_autoscaling_MetricSpec(&metric, &outMetrics[i], s); err != nil {
 				return err
 			}
 		}
+		if out.Spec.Metrics != nil {
+			outMetrics[len(otherMetrics)] = out.Spec.Metrics[0]
+		}
+		out.Spec.Metrics = outMetrics
 		delete(out.Annotations, autoscaling.MetricSpecsAnnotation)
 	}
 

pkg/controller/podautoscaler/BUILD

@@ -20,6 +20,7 @@ go_library(
         "//pkg/api:go_default_library",
         "//pkg/api/v1:go_default_library",
         "//pkg/apis/autoscaling/v1:go_default_library",
+        "//pkg/apis/autoscaling/v2alpha1:go_default_library",
         "//pkg/apis/extensions/v1beta1:go_default_library",
         "//pkg/client/clientset_generated/clientset/typed/autoscaling/v1:go_default_library",
         "//pkg/client/clientset_generated/clientset/typed/core/v1:go_default_library",
@@ -31,6 +32,8 @@ go_library(
         "//vendor:k8s.io/apimachinery/pkg/api/resource",
         "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
         "//vendor:k8s.io/apimachinery/pkg/labels",
+        "//vendor:k8s.io/apimachinery/pkg/runtime",
+        "//vendor:k8s.io/apimachinery/pkg/runtime/schema",
         "//vendor:k8s.io/apimachinery/pkg/util/runtime",
         "//vendor:k8s.io/apimachinery/pkg/util/sets",
         "//vendor:k8s.io/client-go/kubernetes/typed/core/v1",
@@ -53,6 +56,7 @@ go_test(
         "//pkg/api/v1:go_default_library",
         "//pkg/apis/autoscaling/install:go_default_library",
         "//pkg/apis/autoscaling/v1:go_default_library",
+        "//pkg/apis/autoscaling/v2alpha1:go_default_library",
         "//pkg/apis/extensions/install:go_default_library",
         "//pkg/apis/extensions/v1beta1:go_default_library",
         "//pkg/client/clientset_generated/clientset/fake:go_default_library",

pkg/controller/podautoscaler/metrics/BUILD

@@ -17,6 +17,7 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//pkg/api/v1:go_default_library",
+        "//pkg/apis/autoscaling/v2alpha1:go_default_library",
         "//pkg/client/clientset_generated/clientset:go_default_library",
         "//pkg/client/clientset_generated/clientset/typed/core/v1:go_default_library",
         "//vendor:github.com/golang/glog",

pkg/controller/podautoscaler/metrics/metrics_client.go

@@ -29,28 +29,29 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/kubernetes/pkg/api/v1"
+	autoscaling "k8s.io/kubernetes/pkg/apis/autoscaling/v2alpha1"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 	v1core "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1"
 )
 
-// PodResourceInfo contains pod resourcemetric values as a map from pod names to
-// metric values
-type PodResourceInfo map[string]int64
-
-// PodMetricsInfo contains pod resourcemetric values as a map from pod names to
-// metric values
-type PodMetricsInfo map[string]float64
+// PodMetricsInfo contains pod metric values as a map from pod names to
+// metric values (the metric values are expected to be the metric as a milli-value)
+type PodMetricsInfo map[string]int64
 
 // MetricsClient knows how to query a remote interface to retrieve container-level
 // resource metrics as well as pod-level arbitrary metrics
 type MetricsClient interface {
 	// GetResourceMetric gets the given resource metric (and an associated oldest timestamp)
 	// for all pods matching the specified selector in the given namespace
-	GetResourceMetric(resource v1.ResourceName, namespace string, selector labels.Selector) (PodResourceInfo, time.Time, error)
+	GetResourceMetric(resource v1.ResourceName, namespace string, selector labels.Selector) (PodMetricsInfo, time.Time, error)
 
 	// GetRawMetric gets the given metric (and an associated oldest timestamp)
 	// for all pods matching the specified selector in the given namespace
 	GetRawMetric(metricName string, namespace string, selector labels.Selector) (PodMetricsInfo, time.Time, error)
+
+	// GetObjectMetric gets the given metric (and an associated timestamp) for the given
+	// object in the given namespace
+	GetObjectMetric(metricName string, namespace string, objectRef *autoscaling.CrossVersionObjectReference) (int64, time.Time, error)
 }
 
 const (
@@ -80,7 +81,7 @@ func NewHeapsterMetricsClient(client clientset.Interface, namespace, scheme, ser
 	}
 }
 
-func (h *HeapsterMetricsClient) GetResourceMetric(resource v1.ResourceName, namespace string, selector labels.Selector) (PodResourceInfo, time.Time, error) {
+func (h *HeapsterMetricsClient) GetResourceMetric(resource v1.ResourceName, namespace string, selector labels.Selector) (PodMetricsInfo, time.Time, error) {
 	metricPath := fmt.Sprintf("/apis/metrics/v1alpha1/namespaces/%s/pods", namespace)
 	params := map[string]string{"labelSelector": selector.String()}
 
@@ -88,7 +89,7 @@ func (h *HeapsterMetricsClient) GetResourceMetric(resource v1.ResourceName, name
 		ProxyGet(h.heapsterScheme, h.heapsterService, h.heapsterPort, metricPath, params).
 		DoRaw()
 	if err != nil {
-		return nil, time.Time{}, fmt.Errorf("failed to get heapster service: %v", err)
+		return nil, time.Time{}, fmt.Errorf("failed to get pod resource metrics: %v", err)
 	}
 
 	glog.V(4).Infof("Heapster metrics result: %s", string(resultRaw))
@@ -103,7 +104,7 @@ func (h *HeapsterMetricsClient) GetResourceMetric(resource v1.ResourceName, name
 		return nil, time.Time{}, fmt.Errorf("no metrics returned from heapster")
 	}
 
-	res := make(PodResourceInfo, len(metrics.Items))
+	res := make(PodMetricsInfo, len(metrics.Items))
 
 	for _, m := range metrics.Items {
 		podSum := int64(0)
@@ -155,7 +156,7 @@ func (h *HeapsterMetricsClient) GetRawMetric(metricName string, namespace string
 		ProxyGet(h.heapsterScheme, h.heapsterService, h.heapsterPort, metricPath, map[string]string{"start": startTime.Format(time.RFC3339)}).
 		DoRaw()
 	if err != nil {
-		return nil, time.Time{}, fmt.Errorf("failed to get heapster service: %v", err)
+		return nil, time.Time{}, fmt.Errorf("failed to get pod metrics: %v", err)
 	}
 
 	var metrics heapster.MetricResultList
@@ -192,7 +193,11 @@ func (h *HeapsterMetricsClient) GetRawMetric(metricName string, namespace string
 	return res, *timestamp, nil
 }
 
-func collapseTimeSamples(metrics heapster.MetricResult, duration time.Duration) (float64, time.Time, bool) {
+func (h *HeapsterMetricsClient) GetObjectMetric(metricName string, namespace string, objectRef *autoscaling.CrossVersionObjectReference) (int64, time.Time, error) {
+	return 0, time.Time{}, fmt.Errorf("object metrics are not yet supported")
+}
+
+func collapseTimeSamples(metrics heapster.MetricResult, duration time.Duration) (int64, time.Time, bool) {
 	floatSum := float64(0)
 	intSum := int64(0)
 	intSumCount := 0
@@ -217,9 +222,9 @@ func collapseTimeSamples(metrics heapster.MetricResult, duration time.Duration)
 		}
 
 		if newest.FloatValue != nil {
-			return floatSum / float64(floatSumCount), newest.Timestamp, true
+			return int64(floatSum / float64(floatSumCount) * 1000), newest.Timestamp, true
 		} else {
-			return float64(intSum / int64(intSumCount)), newest.Timestamp, true
+			return (intSum * 1000) / int64(intSumCount), newest.Timestamp, true
 		}
 	}
 

pkg/controller/podautoscaler/metrics/metrics_client_test.go

@@ -64,7 +64,6 @@ type metricPoint struct {
 }
 
 type testCase struct {
-	desiredResourceValues PodResourceInfo
 	desiredMetricValues PodMetricsInfo
 	desiredError        error
 
@@ -190,7 +189,7 @@ func buildPod(namespace, podName string, podLabels map[string]string, phase v1.P
 	}
 }
 
-func (tc *testCase) verifyResults(t *testing.T, metrics interface{}, timestamp time.Time, err error) {
+func (tc *testCase) verifyResults(t *testing.T, metrics PodMetricsInfo, timestamp time.Time, err error) {
 	if tc.desiredError != nil {
 		assert.Error(t, err, "there should be an error retrieving the metrics")
 		assert.Contains(t, fmt.Sprintf("%v", err), fmt.Sprintf("%v", tc.desiredError), "the error message should be eas expected")
@@ -199,13 +198,7 @@ func (tc *testCase) verifyResults(t *testing.T, metrics interface{}, timestamp t
 	assert.NoError(t, err, "there should be no error retrieving the metrics")
 	assert.NotNil(t, metrics, "there should be metrics returned")
 
-	if metricsInfo, wasRaw := metrics.(PodMetricsInfo); wasRaw {
-		assert.Equal(t, tc.desiredMetricValues, metricsInfo, "the raw metrics values should be as expected")
-	} else if resourceInfo, wasResource := metrics.(PodResourceInfo); wasResource {
-		assert.Equal(t, tc.desiredResourceValues, resourceInfo, "the resource metrics values be been as expected")
-	} else {
-		assert.False(t, true, "should return either resource metrics info or raw metrics info")
-	}
+	assert.Equal(t, tc.desiredMetricValues, metrics, "the metrics values should be as expected")
 
 	targetTimestamp := fixedTimestamp.Add(time.Duration(tc.targetTimestamp) * time.Minute)
 	assert.True(t, targetTimestamp.Equal(timestamp), fmt.Sprintf("the timestamp should be as expected (%s) but was %s", targetTimestamp, timestamp))
@@ -227,7 +220,7 @@ func (tc *testCase) runTest(t *testing.T) {
 func TestCPU(t *testing.T) {
 	tc := testCase{
 		replicas: 3,
-		desiredResourceValues: PodResourceInfo{
+		desiredMetricValues: PodMetricsInfo{
 			"test-pod-0": 5000, "test-pod-1": 5000, "test-pod-2": 5000,
 		},
 		resourceName:       v1.ResourceCPU,
@@ -241,7 +234,7 @@ func TestQPS(t *testing.T) {
 	tc := testCase{
 		replicas: 3,
 		desiredMetricValues: PodMetricsInfo{
-			"test-pod-0": 10, "test-pod-1": 20, "test-pod-2": 10,
+			"test-pod-0": 10000, "test-pod-1": 20000, "test-pod-2": 10000,
 		},
 		metricName:            "qps",
 		targetTimestamp:       1,
@@ -266,7 +259,7 @@ func TestQpsSumEqualZero(t *testing.T) {
 func TestCPUMoreMetrics(t *testing.T) {
 	tc := testCase{
 		replicas: 5,
-		desiredResourceValues: PodResourceInfo{
+		desiredMetricValues: PodMetricsInfo{
 			"test-pod-0": 5000, "test-pod-1": 5000, "test-pod-2": 5000,
 			"test-pod-3": 5000, "test-pod-4": 5000,
 		},
@@ -280,7 +273,7 @@ func TestCPUMoreMetrics(t *testing.T) {
 func TestCPUMissingMetrics(t *testing.T) {
 	tc := testCase{
 		replicas: 3,
-		desiredResourceValues: PodResourceInfo{
+		desiredMetricValues: PodMetricsInfo{
 			"test-pod-0": 4000,
 		},
 		resourceName:       v1.ResourceCPU,
@@ -326,7 +319,7 @@ func TestQpsEmptyEntries(t *testing.T) {
 		replicas:   3,
 		metricName: "qps",
 		desiredMetricValues: PodMetricsInfo{
-			"test-pod-0": 4000, "test-pod-2": 2000,
+			"test-pod-0": 4000000, "test-pod-2": 2000000,
 		},
 		targetTimestamp:       4,
 		reportedMetricsPoints: [][]metricPoint{{{4000, 4}}, {}, {{2000, 4}}},
@@ -348,7 +341,7 @@ func TestCPUEmptyMetricsForOnePod(t *testing.T) {
 	tc := testCase{
 		replicas:     3,
 		resourceName: v1.ResourceCPU,
-		desiredResourceValues: PodResourceInfo{
+		desiredMetricValues: PodMetricsInfo{
 			"test-pod-0": 100, "test-pod-1": 700,
 		},
 		reportedPodMetrics: [][]int64{{100}, {300, 400}, {}},

pkg/controller/podautoscaler/metrics/utilization.go

@@ -22,10 +22,11 @@ import (
 
 // GetResourceUtilizationRatio takes in a set of metrics, a set of matching requests,
 // and a target utilization percentage, and calcuates the the ratio of
-// desired to actual utilization (returning that and the actual utilization)
-func GetResourceUtilizationRatio(metrics PodResourceInfo, requests map[string]int64, targetUtilization int32) (float64, int32, error) {
+// desired to actual utilization (returning that, the actual utilization, and the raw average value)
+func GetResourceUtilizationRatio(metrics PodMetricsInfo, requests map[string]int64, targetUtilization int32) (utilizationRatio float64, currentUtilization int32, rawAverageValue int64, err error) {
 	metricsTotal := int64(0)
 	requestsTotal := int64(0)
+	numEntries := 0
 
 	for podName, metricValue := range metrics {
 		request, hasRequest := requests[podName]
@@ -36,29 +37,30 @@ func GetResourceUtilizationRatio(metrics PodResourceInfo, requests map[string]in
 
 		metricsTotal += metricValue
 		requestsTotal += request
+		numEntries++
 	}
 
 	// if the set of requests is completely disjoint from the set of metrics,
 	// then we could have an issue where the requests total is zero
 	if requestsTotal == 0 {
-		return 0, 0, fmt.Errorf("no metrics returned matched known pods")
+		return 0, 0, 0, fmt.Errorf("no metrics returned matched known pods")
 	}
 
-	currentUtilization := int32((metricsTotal * 100) / requestsTotal)
+	currentUtilization = int32((metricsTotal * 100) / requestsTotal)
 
-	return float64(currentUtilization) / float64(targetUtilization), currentUtilization, nil
+	return float64(currentUtilization) / float64(targetUtilization), currentUtilization, metricsTotal / int64(numEntries), nil
 }
 
 // GetMetricUtilizationRatio takes in a set of metrics and a target utilization value,
 // and calcuates the ratio of desired to actual utilization
 // (returning that and the actual utilization)
-func GetMetricUtilizationRatio(metrics PodMetricsInfo, targetUtilization float64) (float64, float64) {
-	metricsTotal := float64(0)
+func GetMetricUtilizationRatio(metrics PodMetricsInfo, targetUtilization int64) (utilizationRatio float64, currentUtilization int64) {
+	metricsTotal := int64(0)
 	for _, metricValue := range metrics {
 		metricsTotal += metricValue
 	}
 
-	currentUtilization := metricsTotal / float64(len(metrics))
+	currentUtilization = metricsTotal / int64(len(metrics))
 
-	return currentUtilization / targetUtilization, currentUtilization
+	return float64(currentUtilization) / float64(targetUtilization), currentUtilization
 }

pkg/controller/podautoscaler/replica_calculator_test.go

@@ -51,14 +51,15 @@ type resourceInfo struct {
 
 	targetUtilization   int32
 	expectedUtilization int32
+	expectedValue       int64
 }
 
 type metricInfo struct {
 	name   string
 	levels []float64
 
-	targetUtilization   float64
-	expectedUtilization float64
+	targetUtilization   int64
+	expectedUtilization int64
 }
 
 type replicaCalcTestCase struct {
@@ -77,6 +78,7 @@ type replicaCalcTestCase struct {
 const (
 	testNamespace       = "test-namespace"
 	podNamePrefix       = "test-pod"
+	numContainersPerPod = 2
 )
 
 func (tc *replicaCalcTestCase) prepareTestClient(t *testing.T) *fake.Clientset {
@@ -145,24 +147,18 @@ func (tc *replicaCalcTestCase) prepareTestClient(t *testing.T) *fake.Clientset {
 						Namespace: testNamespace,
 					},
 					Timestamp:  unversioned.Time{Time: tc.timestamp},
-					Containers: []metricsapi.ContainerMetrics{
-						{
-							Name: "container1",
-							Usage: v1.ResourceList{
-								v1.ResourceName(tc.resource.name): *resource.NewMilliQuantity(
-									int64(resValue),
-									resource.DecimalSI),
-							},
-						},
-						{
-							Name: "container2",
+					Containers: make([]metricsapi.ContainerMetrics, numContainersPerPod),
+				}
+
+				for i := 0; i < numContainersPerPod; i++ {
+					podMetric.Containers[i] = metricsapi.ContainerMetrics{
+						Name: fmt.Sprintf("container%v", i),
 						Usage: v1.ResourceList{
 							v1.ResourceName(tc.resource.name): *resource.NewMilliQuantity(
 								int64(resValue),
 								resource.DecimalSI),
 						},
-						},
-					},
+					}
 				}
 				metrics.Items = append(metrics.Items, podMetric)
 			}
@@ -228,7 +224,7 @@ func (tc *replicaCalcTestCase) runTest(t *testing.T) {
 	}
 
 	if tc.resource != nil {
-		outReplicas, outUtilization, outTimestamp, err := replicaCalc.GetResourceReplicas(tc.currentReplicas, tc.resource.targetUtilization, tc.resource.name, testNamespace, selector)
+		outReplicas, outUtilization, outRawValue, outTimestamp, err := replicaCalc.GetResourceReplicas(tc.currentReplicas, tc.resource.targetUtilization, tc.resource.name, testNamespace, selector)
 
 		if tc.expectedError != nil {
 			require.Error(t, err, "there should be an error calculating the replica count")
@@ -238,6 +234,7 @@ func (tc *replicaCalcTestCase) runTest(t *testing.T) {
 		require.NoError(t, err, "there should not have been an error calculating the replica count")
 		assert.Equal(t, tc.expectedReplicas, outReplicas, "replicas should be as expected")
 		assert.Equal(t, tc.resource.expectedUtilization, outUtilization, "utilization should be as expected")
+		assert.Equal(t, tc.resource.expectedValue, outRawValue, "raw value should be as expected")
 		assert.True(t, tc.timestamp.Equal(outTimestamp), "timestamp should be as expected")
 
 	} else {
@@ -250,7 +247,7 @@ func (tc *replicaCalcTestCase) runTest(t *testing.T) {
 		}
 		require.NoError(t, err, "there should not have been an error calculating the replica count")
 		assert.Equal(t, tc.expectedReplicas, outReplicas, "replicas should be as expected")
-		assert.InDelta(t, tc.metric.expectedUtilization, 0.1, outUtilization, "utilization should be as expected")
+		assert.Equal(t, tc.metric.expectedUtilization, outUtilization, "utilization should be as expected")
 		assert.True(t, tc.timestamp.Equal(outTimestamp), "timestamp should be as expected")
 	}
 }
@@ -282,6 +279,7 @@ func TestReplicaCalcScaleUp(t *testing.T) {
 
 			targetUtilization:   30,
 			expectedUtilization: 50,
+			expectedValue:       numContainersPerPod * 500,
 		},
 	}
 	tc.runTest(t)
@@ -299,6 +297,7 @@ func TestReplicaCalcScaleUpUnreadyLessScale(t *testing.T) {
 
 			targetUtilization:   30,
 			expectedUtilization: 60,
+			expectedValue:       numContainersPerPod * 600,
 		},
 	}
 	tc.runTest(t)
@@ -316,6 +315,7 @@ func TestReplicaCalcScaleUpUnreadyNoScale(t *testing.T) {
 
 			targetUtilization:   30,
 			expectedUtilization: 40,
+			expectedValue:       numContainersPerPod * 400,
 		},
 	}
 	tc.runTest(t)
@@ -328,8 +328,8 @@ func TestReplicaCalcScaleUpCM(t *testing.T) {
 		metric: &metricInfo{
 			name:                "qps",
 			levels:              []float64{20.0, 10.0, 30.0},
-			targetUtilization:   15.0,
-			expectedUtilization: 20.0,
+			targetUtilization:   15000,
+			expectedUtilization: 20000,
 		},
 	}
 	tc.runTest(t)
@@ -343,8 +343,8 @@ func TestReplicaCalcScaleUpCMUnreadyLessScale(t *testing.T) {
 		metric: &metricInfo{
 			name:                "qps",
 			levels:              []float64{50.0, 10.0, 30.0},
-			targetUtilization:   15.0,
-			expectedUtilization: 30.0,
+			targetUtilization:   15000,
+			expectedUtilization: 30000,
 		},
 	}
 	tc.runTest(t)
@@ -358,8 +358,8 @@ func TestReplicaCalcScaleUpCMUnreadyNoScaleWouldScaleDown(t *testing.T) {
 		metric: &metricInfo{
 			name:                "qps",
 			levels:              []float64{50.0, 15.0, 30.0},
-			targetUtilization:   15.0,
-			expectedUtilization: 15.0,
+			targetUtilization:   15000,
+			expectedUtilization: 15000,
 		},
 	}
 	tc.runTest(t)
@@ -376,6 +376,7 @@ func TestReplicaCalcScaleDown(t *testing.T) {
 
 			targetUtilization:   50,
 			expectedUtilization: 28,
+			expectedValue:       numContainersPerPod * 280,
 		},
 	}
 	tc.runTest(t)
@@ -388,8 +389,8 @@ func TestReplicaCalcScaleDownCM(t *testing.T) {
 		metric: &metricInfo{
 			name:                "qps",
 			levels:              []float64{12.0, 12.0, 12.0, 12.0, 12.0},
-			targetUtilization:   20.0,
-			expectedUtilization: 12.0,
+			targetUtilization:   20000,
+			expectedUtilization: 12000,
 		},
 	}
 	tc.runTest(t)
@@ -407,6 +408,7 @@ func TestReplicaCalcScaleDownIgnoresUnreadyPods(t *testing.T) {
 
 			targetUtilization:   50,
 			expectedUtilization: 30,
+			expectedValue:       numContainersPerPod * 300,
 		},
 	}
 	tc.runTest(t)
@@ -423,6 +425,7 @@ func TestReplicaCalcTolerance(t *testing.T) {
 
 			targetUtilization:   100,
 			expectedUtilization: 102,
+			expectedValue:       numContainersPerPod * 1020,
 		},
 	}
 	tc.runTest(t)
@@ -435,8 +438,8 @@ func TestReplicaCalcToleranceCM(t *testing.T) {
 		metric: &metricInfo{
 			name:                "qps",
 			levels:              []float64{20.0, 21.0, 21.0},
-			targetUtilization:   20.0,
-			expectedUtilization: 20.66666,
+			targetUtilization:   20000,
+			expectedUtilization: 20666,
 		},
 	}
 	tc.runTest(t)
@@ -452,6 +455,7 @@ func TestReplicaCalcSuperfluousMetrics(t *testing.T) {
 			levels:              []int64{4000, 9500, 3000, 7000, 3200, 2000},
 			targetUtilization:   100,
 			expectedUtilization: 587,
+			expectedValue:       numContainersPerPod * 5875,
 		},
 	}
 	tc.runTest(t)
@@ -468,6 +472,7 @@ func TestReplicaCalcMissingMetrics(t *testing.T) {
 
 			targetUtilization:   100,
 			expectedUtilization: 24,
+			expectedValue:       495, // numContainersPerPod * 247, for sufficiently large values of 247
 		},
 	}
 	tc.runTest(t)
@@ -514,6 +519,7 @@ func TestReplicaCalcMissingMetricsNoChangeEq(t *testing.T) {
 
 			targetUtilization:   100,
 			expectedUtilization: 100,
+			expectedValue:       numContainersPerPod * 1000,
 		},
 	}
 	tc.runTest(t)
@@ -530,6 +536,7 @@ func TestReplicaCalcMissingMetricsNoChangeGt(t *testing.T) {
 
 			targetUtilization:   100,
 			expectedUtilization: 190,
+			expectedValue:       numContainersPerPod * 1900,
 		},
 	}
 	tc.runTest(t)
@@ -546,6 +553,7 @@ func TestReplicaCalcMissingMetricsNoChangeLt(t *testing.T) {
 
 			targetUtilization:   100,
 			expectedUtilization: 60,
+			expectedValue:       numContainersPerPod * 600,
 		},
 	}
 	tc.runTest(t)
@@ -563,6 +571,7 @@ func TestReplicaCalcMissingMetricsUnreadyNoChange(t *testing.T) {
 
 			targetUtilization:   50,
 			expectedUtilization: 45,
+			expectedValue:       numContainersPerPod * 450,
 		},
 	}
 	tc.runTest(t)
@@ -580,6 +589,7 @@ func TestReplicaCalcMissingMetricsUnreadyScaleUp(t *testing.T) {
 
 			targetUtilization:   50,
 			expectedUtilization: 200,
+			expectedValue:       numContainersPerPod * 2000,
 		},
 	}
 	tc.runTest(t)
@@ -597,6 +607,7 @@ func TestReplicaCalcMissingMetricsUnreadyScaleDown(t *testing.T) {
 
 			targetUtilization:   50,
 			expectedUtilization: 10,
+			expectedValue:       numContainersPerPod * 100,
 		},
 	}
 	tc.runTest(t)
@@ -658,6 +669,7 @@ func TestReplicaCalcComputedToleranceAlgImplementation(t *testing.T) {
 
 			targetUtilization:   finalCpuPercentTarget,
 			expectedUtilization: int32(totalUsedCPUOfAllPods*100) / totalRequestedCPUOfAllPods,
+			expectedValue:       numContainersPerPod * totalUsedCPUOfAllPods / 10,
 		},
 	}