Add retries in cluster-autoscaler e2e

test/e2e/autoscaling/cluster_size_autoscaling.go

@@ -674,6 +674,7 @@ func setMigSizes(sizes map[string]int) {
 
 func makeNodeUnschedulable(c clientset.Interface, node *v1.Node) error {
 	By(fmt.Sprintf("Taint node %s", node.Name))
+	for j := 0; j < 3; j++ {
 		freshNode, err := c.Core().Nodes().Get(node.Name, metav1.GetOptions{})
 		if err != nil {
 			return err
@@ -689,11 +690,20 @@ func makeNodeUnschedulable(c clientset.Interface, node *v1.Node) error {
 			Effect: v1.TaintEffectNoSchedule,
 		})
 		_, err = c.Core().Nodes().Update(freshNode)
+		if err == nil {
+			return nil
+		}
+		if !errors.IsConflict(err) {
 			return err
+		}
+		glog.Warningf("Got 409 conflict when trying to taint node, retries left: %v", 3-j)
+	}
+	return fmt.Errorf("Failed to taint node in allowed number of retries")
 }
 
 func makeNodeSchedulable(c clientset.Interface, node *v1.Node) error {
 	By(fmt.Sprintf("Remove taint from node %s", node.Name))
+	for j := 0; j < 3; j++ {
 		freshNode, err := c.Core().Nodes().Get(node.Name, metav1.GetOptions{})
 		if err != nil {
 			return err
@@ -705,12 +715,20 @@ func makeNodeSchedulable(c clientset.Interface, node *v1.Node) error {
 			}
 		}
 
-	if len(newTaints) != len(freshNode.Spec.Taints) {
+		if len(newTaints) == len(freshNode.Spec.Taints) {
+			return nil
+		}
 		freshNode.Spec.Taints = newTaints
 		_, err = c.Core().Nodes().Update(freshNode)
+		if err == nil {
+			return nil
+		}
+		if !errors.IsConflict(err) {
 			return err
 		}
-	return nil
+		glog.Warningf("Got 409 conflict when trying to taint node, retries left: %v", 3-j)
+	}
+	return fmt.Errorf("Failed to remove taint from node in allowed number of retries")
 }
 
 // Creat an RC running a given number of pods on each node without adding any constraint forcing