Adjust the sleep time before tearing down e2e clusters and add a rough formula

hack/jenkins/e2e.sh

@@ -224,8 +224,10 @@ fi
 if [[ "${E2E_DOWN,,}" == "true" ]]; then
     # Sleep before deleting the cluster to give the controller manager time to
     # delete any cloudprovider resources still around from the last test.
-    # 60 seconds was chosen as an arbitrary bound for how long it should take
-    # to delete load balancer resources.
-    sleep 60
+    # This is calibrated to allow enough time for 3 attempts to delete the
+    # resources. Each attempt is allocated 5 seconds for requests to the
+    # cloudprovider plus the processingRetryInterval from servicecontroller.go
+    # for the wait between attempts.
+    sleep 30
     go run ./hack/e2e.go ${E2E_OPT} -v --down
 fi

pkg/cloudprovider/servicecontroller/servicecontroller.go

@@ -38,6 +38,11 @@ import (
 const (
 	workerGoroutines = 10
 
+	// How long to wait before retrying the processing of a service change.
+	// If this changes, the sleep in hack/jenkins/e2e.sh before downing a cluster
+	// should be changed appropriately.
+	processingRetryInterval = 5 * time.Second
+
 	clientRetryCount    = 5
 	clientRetryInterval = 5 * time.Second
 
@@ -151,7 +156,7 @@ func (s *ServiceController) watchServices(serviceQueue *cache.DeltaFIFO) {
 		if shouldRetry {
 			// Add the failed service back to the queue so we'll retry it.
 			glog.Errorf("Failed to process service delta. Retrying: %v", err)
-			time.Sleep(5 * time.Second)
+			time.Sleep(processingRetryInterval)
 			serviceQueue.AddIfNotPresent(deltas)
 		} else if err != nil {
 			util.HandleError(fmt.Errorf("Failed to process service delta. Not retrying: %v", err))