Merge pull request #59830 from khenidak/az-ratelimit

Automatic merge from submit-queue (batch tested with PRs 59939, 59830). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Azure - ARM Read/Write rate limiting **What this PR does / why we need it**: Azure cloud provider currently runs with: 1. Single ARM rate limiter for both `read [put/post/delete]` and `write` operations, while ARM provide [different rates for read/write] (https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-manager-request-limits). This causes write operation to stop even if there is available write request quotas. 2. Cloud provider uses rate limiter's `Accept()` instead of `TryAccept()` This causes control loop to wait for prolonged tike `in case of no request quota available` for **all** requests even for those does not require ARM interaction. A case for that the `Service` control loop will wait for a prolonged time trying to create `LoadBalancer` service even though it can fail and work on the next service which is `ClusterIP`. This PR moves cloud provider tp `TryAccept()` **Which issue(s) this PR fixes**: Fixes # https://github.com/kubernetes/kubernetes/issues/58770 **Special notes for your reviewer**: `n/a` **Release note**: ```release-note - Separate current ARM rate limiter into read/write - Improve control over how ARM rate limiter is used within Azure cloud provider ``` cc @jackfrancis (need your help carefully reviewing this one) @brendanburns @jdumars

Merge pull request #59830 from khenidak/az-ratelimit
271c267f · Kubernetes Submit Queue · GitHub · 281cb007 · 38a9fc33 · 271c267f
Unverified Commit 271c267f authored Feb 15, 2018 by Kubernetes Submit Queue Committed by GitHub Feb 15, 2018
Expand all Hide whitespace changes
Inline Side-by-side

Showing with 29 additions and 3 deletions

azure.go pkg/cloudprovider/providers/azure/azure.go +29 -3

azure_client.go pkg/cloudprovider/providers/azure/azure_client.go +0 -0

No files found.
--- a/pkg/cloudprovider/providers/azure/azure.go
+++ b/pkg/cloudprovider/providers/azure/azure.go
@@ -97,10 +97,14 @@ type Config struct {
 	CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter" yaml:"cloudProviderBackoffJitter"`
 	// Enable rate limiting
 	CloudProviderRateLimit bool `json:"cloudProviderRateLimit" yaml:"cloudProviderRateLimit"`
-	// Rate limit QPS
+	// Rate limit QPS (Read)
 	CloudProviderRateLimitQPS float32 `json:"cloudProviderRateLimitQPS" yaml:"cloudProviderRateLimitQPS"`
 	// Rate limit Bucket Size
 	CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket" yaml:"cloudProviderRateLimitBucket"`
+	// Rate limit QPS (Write)
+	CloudProviderRateLimitQPSWrite float32 `json:"cloudProviderRateLimitQPSWrite" yaml:"cloudProviderRateLimitQPSWrite"`
+	// Rate limit Bucket Size
+	CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite" yaml:"cloudProviderRateLimitBucketWrite"`

 	// Use instance metadata service where possible
 	UseInstanceMetadata bool `json:"useInstanceMetadata" yaml:"useInstanceMetadata"`
@@ -168,6 +172,10 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {

 	// operationPollRateLimiter.Accept() is a no-op if rate limits are configured off.
 	operationPollRateLimiter := flowcontrol.NewFakeAlwaysRateLimiter()
+	operationPollRateLimiterWrite := flowcontrol.NewFakeAlwaysRateLimiter()
+
+	// If reader is provided (and no writer) we will
+	// use the same value for both.
 	if config.CloudProviderRateLimit {
 		// Assign rate limit defaults if no configuration was passed in
 		if config.CloudProviderRateLimitQPS == 0 {
@@ -176,19 +184,37 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
 		if config.CloudProviderRateLimitBucket == 0 {
 			config.CloudProviderRateLimitBucket = rateLimitBucketDefault
 		}
+		if config.CloudProviderRateLimitQPSWrite == 0 {
+			config.CloudProviderRateLimitQPSWrite = rateLimitQPSDefault
+		}
+		if config.CloudProviderRateLimitBucketWrite == 0 {
+			config.CloudProviderRateLimitBucketWrite = rateLimitBucketDefault
+		}
+
 		operationPollRateLimiter = flowcontrol.NewTokenBucketRateLimiter(
 			config.CloudProviderRateLimitQPS,
 			config.CloudProviderRateLimitBucket)
-		glog.V(2).Infof("Azure cloudprovider using rate limit config: QPS=%g, bucket=%d",
+
+		operationPollRateLimiterWrite = flowcontrol.NewTokenBucketRateLimiter(
+			config.CloudProviderRateLimitQPSWrite,
+			config.CloudProviderRateLimitBucketWrite)
+
+		glog.V(2).Infof("Azure cloudprovider (read ops) using rate limit config: QPS=%g, bucket=%d",
 			config.CloudProviderRateLimitQPS,
 			config.CloudProviderRateLimitBucket)
+
+		glog.V(2).Infof("Azure cloudprovider (write ops) using rate limit config: QPS=%g, bucket=%d",
+			config.CloudProviderRateLimitQPSWrite,
+			config.CloudProviderRateLimitBucketWrite)
+
 	}

 	azClientConfig := &azClientConfig{
 		subscriptionID:          config.SubscriptionID,
 		resourceManagerEndpoint: env.ResourceManagerEndpoint,
 		servicePrincipalToken:   servicePrincipalToken,
-		rateLimiter:             operationPollRateLimiter,
+		rateLimiterReader:       operationPollRateLimiter,
+		rateLimiterWriter:       operationPollRateLimiterWrite,
 	}
 	az := Cloud{
 		Config:      *config,

--- a/pkg/cloudprovider/providers/azure/azure_client.go
+++ b/pkg/cloudprovider/providers/azure/azure_client.go