Commit f76ef295 authored by Jack Francis's avatar Jack Francis

backing off az.getIPForMachine in az.NodeAddresses

also rate limiting the call to az.getVirtualMachine inside az.getIPForMachine
parent 2525ef99
...@@ -59,6 +59,22 @@ func (az *Cloud) VirtualMachineClientGetWithRetry(resourceGroup, vmName string, ...@@ -59,6 +59,22 @@ func (az *Cloud) VirtualMachineClientGetWithRetry(resourceGroup, vmName string,
return machine, err return machine, err
} }
// GetIPForMachineWithRetry invokes az.getIPForMachine with exponential backoff retry
func (az *Cloud) GetIPForMachineWithRetry(name types.NodeName) (string, error) {
var ip string
err := wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
var retryErr error
ip, retryErr = az.getIPForMachine(name)
if retryErr != nil {
glog.Errorf("backoff: failure, will retry,err=%v", retryErr)
return false, nil
}
glog.V(2).Infof("backoff: success")
return true, nil
})
return ip, err
}
// CreateOrUpdateSGWithRetry invokes az.SecurityGroupsClient.CreateOrUpdate with exponential backoff retry // CreateOrUpdateSGWithRetry invokes az.SecurityGroupsClient.CreateOrUpdate with exponential backoff retry
func (az *Cloud) CreateOrUpdateSGWithRetry(sg network.SecurityGroup) error { func (az *Cloud) CreateOrUpdateSGWithRetry(sg network.SecurityGroup) error {
return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) { return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
......
...@@ -41,8 +41,17 @@ func (az *Cloud) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) { ...@@ -41,8 +41,17 @@ func (az *Cloud) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) {
} }
ip, err := az.getIPForMachine(name) ip, err := az.getIPForMachine(name)
if err != nil { if err != nil {
glog.Errorf("error: az.NodeAddresses, az.getIPForMachine(%s), err=%v", name, err) if az.CloudProviderBackoff {
return nil, err glog.V(2).Infof("NodeAddresses(%s) backing off", name)
ip, err = az.GetIPForMachineWithRetry(name)
if err != nil {
glog.V(2).Infof("NodeAddresses(%s) abort backoff", name)
return nil, err
}
} else {
glog.Errorf("error: az.NodeAddresses, az.getIPForMachine(%s), err=%v", name, err)
return nil, err
}
} }
return []v1.NodeAddress{ return []v1.NodeAddress{
......
...@@ -249,6 +249,7 @@ outer: ...@@ -249,6 +249,7 @@ outer:
} }
func (az *Cloud) getIPForMachine(nodeName types.NodeName) (string, error) { func (az *Cloud) getIPForMachine(nodeName types.NodeName) (string, error) {
az.operationPollRateLimiter.Accept()
machine, exists, err := az.getVirtualMachine(nodeName) machine, exists, err := az.getVirtualMachine(nodeName)
if !exists { if !exists {
return "", cloudprovider.InstanceNotFound return "", cloudprovider.InstanceNotFound
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment