Skip to content

K
k3s
Commit 83ca86cd authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #49332 from shyamjvs/framework-fix

Automatic merge from submit-queue Change e2e-framework behavior to retry retriable API errors Fixes https://github.com/kubernetes/kubernetes/issues/46195 We shouldn't return with failure just because api call failed during polling. Happens from time to time, for e.g https://k8s-gubernator.appspot.com/build/kubernetes-jenkins/logs/ci-kubernetes-kubemark-5-gce/16896 /cc @wojtek-t
parents 3a0d8f8f 6577ece5
Hide whitespace changes
Inline Side-by-side
Showing with 87 additions and 15 deletions
test/e2e/framework/util.go
View file @ 83ca86cd
...@@ -505,7 +505,10 @@ func WaitForPodsSuccess(c clientset.Interface, ns string, successPodLabels map[s ...@@ -505,7 +505,10 @@ func WaitForPodsSuccess(c clientset.Interface, ns string, successPodLabels map[s
podList, err := c.Core().Pods(ns).List(metav1.ListOptions{LabelSelector: successPodSelector.String()}) podList, err := c.Core().Pods(ns).List(metav1.ListOptions{LabelSelector: successPodSelector.String()})
if err != nil { if err != nil {
Logf("Error getting pods in namespace %q: %v", ns, err) Logf("Error getting pods in namespace %q: %v", ns, err)
return false, nil if IsRetryableAPIError(err) {
return false, nil
}
return false, err
} }
if len(podList.Items) == 0 { if len(podList.Items) == 0 {
Logf("Waiting for pods to enter Success, but no pods in %q match label %v", ns, successPodLabels) Logf("Waiting for pods to enter Success, but no pods in %q match label %v", ns, successPodLabels)
...@@ -564,7 +567,10 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedN ...@@ -564,7 +567,10 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedN
rcList, err := c.Core().ReplicationControllers(ns).List(metav1.ListOptions{}) rcList, err := c.Core().ReplicationControllers(ns).List(metav1.ListOptions{})
if err != nil { if err != nil {
Logf("Error getting replication controllers in namespace '%s': %v", ns, err) Logf("Error getting replication controllers in namespace '%s': %v", ns, err)
return false, nil if IsRetryableAPIError(err) {
return false, nil
}
return false, err
} }
for _, rc := range rcList.Items { for _, rc := range rcList.Items {
replicas += *rc.Spec.Replicas replicas += *rc.Spec.Replicas
...@@ -574,7 +580,10 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedN ...@@ -574,7 +580,10 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedN
rsList, err := c.Extensions().ReplicaSets(ns).List(metav1.ListOptions{}) rsList, err := c.Extensions().ReplicaSets(ns).List(metav1.ListOptions{})
if err != nil { if err != nil {
Logf("Error getting replication sets in namespace %q: %v", ns, err) Logf("Error getting replication sets in namespace %q: %v", ns, err)
return false, nil if IsRetryableAPIError(err) {
return false, nil
}
return false, err
} }
for _, rs := range rsList.Items { for _, rs := range rsList.Items {
replicas += *rs.Spec.Replicas replicas += *rs.Spec.Replicas
...@@ -584,7 +593,10 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedN ...@@ -584,7 +593,10 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedN
podList, err := c.Core().Pods(ns).List(metav1.ListOptions{}) podList, err := c.Core().Pods(ns).List(metav1.ListOptions{})
if err != nil { if err != nil {
Logf("Error getting pods in namespace '%s': %v", ns, err) Logf("Error getting pods in namespace '%s': %v", ns, err)
return false, nil if IsRetryableAPIError(err) {
return false, nil
}
return false, err
} }
nOk := int32(0) nOk := int32(0)
notReady := int32(0) notReady := int32(0)
...@@ -1435,6 +1447,9 @@ func WaitForPodToDisappear(c clientset.Interface, ns, podName string, label labe ...@@ -1435,6 +1447,9 @@ func WaitForPodToDisappear(c clientset.Interface, ns, podName string, label labe
options := metav1.ListOptions{LabelSelector: label.String()} options := metav1.ListOptions{LabelSelector: label.String()}
pods, err := c.Core().Pods(ns).List(options) pods, err := c.Core().Pods(ns).List(options)
if err != nil { if err != nil {
if IsRetryableAPIError(err) {
return false, nil
}
return false, err return false, err
} }
found := false found := false
...@@ -1464,6 +1479,9 @@ func WaitForService(c clientset.Interface, namespace, name string, exist bool, i ...@@ -1464,6 +1479,9 @@ func WaitForService(c clientset.Interface, namespace, name string, exist bool, i
case apierrs.IsNotFound(err): case apierrs.IsNotFound(err):
Logf("Service %s in namespace %s disappeared.", name, namespace) Logf("Service %s in namespace %s disappeared.", name, namespace)
return !exist, nil return !exist, nil
case !IsRetryableAPIError(err):
Logf("Non-retryable failure while getting service.")
return false, err
default: default:
Logf("Get service %s in namespace %s failed: %v", name, namespace, err) Logf("Get service %s in namespace %s failed: %v", name, namespace, err)
return false, nil return false, nil
...@@ -1488,6 +1506,9 @@ func WaitForServiceWithSelector(c clientset.Interface, namespace string, selecto ...@@ -1488,6 +1506,9 @@ func WaitForServiceWithSelector(c clientset.Interface, namespace string, selecto
case len(services.Items) == 0: case len(services.Items) == 0:
Logf("Service with %s in namespace %s disappeared.", selector.String(), namespace) Logf("Service with %s in namespace %s disappeared.", selector.String(), namespace)
return !exist, nil return !exist, nil
case !IsRetryableAPIError(err):
Logf("Non-retryable failure while listing service.")
return false, err
default: default:
Logf("List service with %s in namespace %s failed: %v", selector.String(), namespace, err) Logf("List service with %s in namespace %s failed: %v", selector.String(), namespace, err)
return false, nil return false, nil
...@@ -2315,9 +2336,15 @@ func waitListSchedulableNodesOrDie(c clientset.Interface) *v1.NodeList { ...@@ -2315,9 +2336,15 @@ func waitListSchedulableNodesOrDie(c clientset.Interface) *v1.NodeList {
nodes, err = c.Core().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{ nodes, err = c.Core().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{
"spec.unschedulable": "false", "spec.unschedulable": "false",
}.AsSelector().String()}) }.AsSelector().String()})
return err == nil, nil if err != nil {
if IsRetryableAPIError(err) {
return false, nil
}
return false, err
}
return true, nil
}) != nil { }) != nil {
ExpectNoError(err, "Timed out while listing nodes for e2e cluster.") ExpectNoError(err, "Non-retryable failure or timed out while listing nodes for e2e cluster.")
} }
return nodes return nodes
} }
...@@ -2392,8 +2419,10 @@ func WaitForAllNodesSchedulable(c clientset.Interface, timeout time.Duration) er ...@@ -2392,8 +2419,10 @@ func WaitForAllNodesSchedulable(c clientset.Interface, timeout time.Duration) er
nodes, err := c.Core().Nodes().List(opts) nodes, err := c.Core().Nodes().List(opts)
if err != nil { if err != nil {
Logf("Unexpected error listing nodes: %v", err) Logf("Unexpected error listing nodes: %v", err)
// Ignore the error here - it will be retried. if IsRetryableAPIError(err) {
return false, nil return false, nil
}
return false, err
} }
for i := range nodes.Items { for i := range nodes.Items {
node := &nodes.Items[i] node := &nodes.Items[i]
...@@ -2538,7 +2567,12 @@ func NodeHasTaint(c clientset.Interface, nodeName string, taint *v1.Taint) (bool ...@@ -2538,7 +2567,12 @@ func NodeHasTaint(c clientset.Interface, nodeName string, taint *v1.Taint) (bool
func AddOrUpdateAvoidPodOnNode(c clientset.Interface, nodeName string, avoidPods v1.AvoidPods) { func AddOrUpdateAvoidPodOnNode(c clientset.Interface, nodeName string, avoidPods v1.AvoidPods) {
err := wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) { err := wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) {
node, err := c.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) node, err := c.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
ExpectNoError(err) if err != nil {
if IsRetryableAPIError(err) {
return false, nil
}
return false, err
}
taintsData, err := json.Marshal(avoidPods) taintsData, err := json.Marshal(avoidPods)
ExpectNoError(err) ExpectNoError(err)
...@@ -2564,7 +2598,12 @@ func AddOrUpdateAvoidPodOnNode(c clientset.Interface, nodeName string, avoidPods ...@@ -2564,7 +2598,12 @@ func AddOrUpdateAvoidPodOnNode(c clientset.Interface, nodeName string, avoidPods
func RemoveAvoidPodsOffNode(c clientset.Interface, nodeName string) { func RemoveAvoidPodsOffNode(c clientset.Interface, nodeName string) {
err := wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) { err := wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) {
node, err := c.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) node, err := c.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
ExpectNoError(err) if err != nil {
if IsRetryableAPIError(err) {
return false, nil
}
return false, err
}
if node.Annotations == nil { if node.Annotations == nil {
return true, nil return true, nil
...@@ -2668,7 +2707,12 @@ func WaitForPodsWithLabel(c clientset.Interface, ns string, label labels.Selecto ...@@ -2668,7 +2707,12 @@ func WaitForPodsWithLabel(c clientset.Interface, ns string, label labels.Selecto
for t := time.Now(); time.Since(t) < PodListTimeout; time.Sleep(Poll) { for t := time.Now(); time.Since(t) < PodListTimeout; time.Sleep(Poll) {
options := metav1.ListOptions{LabelSelector: label.String()} options := metav1.ListOptions{LabelSelector: label.String()}
pods, err = c.Core().Pods(ns).List(options) pods, err = c.Core().Pods(ns).List(options)
Expect(err).NotTo(HaveOccurred()) if err != nil {
if IsRetryableAPIError(err) {
continue
}
return
}
if len(pods.Items) > 0 { if len(pods.Items) > 0 {
break break
} }
...@@ -2688,7 +2732,10 @@ func WaitForPodsWithLabelRunningReady(c clientset.Interface, ns string, label la ...@@ -2688,7 +2732,10 @@ func WaitForPodsWithLabelRunningReady(c clientset.Interface, ns string, label la
pods, err := WaitForPodsWithLabel(c, ns, label) pods, err := WaitForPodsWithLabel(c, ns, label)
if err != nil { if err != nil {
Logf("Failed to list pods: %v", err) Logf("Failed to list pods: %v", err)
return false, nil if IsRetryableAPIError(err) {
return false, nil
}
return false, err
} }
current = 0 current = 0
for _, pod := range pods.Items { for _, pod := range pods.Items {
...@@ -3012,6 +3059,9 @@ func UpdateDaemonSetWithRetries(c clientset.Interface, namespace, name string, a ...@@ -3012,6 +3059,9 @@ func UpdateDaemonSetWithRetries(c clientset.Interface, namespace, name string, a
var updateErr error var updateErr error
pollErr := wait.PollImmediate(10*time.Millisecond, 1*time.Minute, func() (bool, error) { pollErr := wait.PollImmediate(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
if ds, err = daemonsets.Get(name, metav1.GetOptions{}); err != nil { if ds, err = daemonsets.Get(name, metav1.GetOptions{}); err != nil {
if IsRetryableAPIError(err) {
return false, nil
}
return false, err return false, err
} }
// Apply the update, then attempt to push it to the apiserver. // Apply the update, then attempt to push it to the apiserver.
...@@ -3240,7 +3290,10 @@ func CreateExecPodOrFail(client clientset.Interface, ns, generateName string, tw ...@@ -3240,7 +3290,10 @@ func CreateExecPodOrFail(client clientset.Interface, ns, generateName string, tw
err = wait.PollImmediate(Poll, 5*time.Minute, func() (bool, error) { err = wait.PollImmediate(Poll, 5*time.Minute, func() (bool, error) {
retrievedPod, err := client.Core().Pods(execPod.Namespace).Get(created.Name, metav1.GetOptions{}) retrievedPod, err := client.Core().Pods(execPod.Namespace).Get(created.Name, metav1.GetOptions{})
if err != nil { if err != nil {
return false, nil if IsRetryableAPIError(err) {
return false, nil
}
return false, err
} }
return retrievedPod.Status.Phase == v1.PodRunning, nil return retrievedPod.Status.Phase == v1.PodRunning, nil
}) })
...@@ -3509,6 +3562,9 @@ func AllNodesReady(c clientset.Interface, timeout time.Duration) error { ...@@ -3509,6 +3562,9 @@ func AllNodesReady(c clientset.Interface, timeout time.Duration) error {
// It should be OK to list unschedulable Nodes here. // It should be OK to list unschedulable Nodes here.
nodes, err := c.Core().Nodes().List(metav1.ListOptions{}) nodes, err := c.Core().Nodes().List(metav1.ListOptions{})
if err != nil { if err != nil {
if IsRetryableAPIError(err) {
return false, nil
}
return false, err return false, err
} }
for i := range nodes.Items { for i := range nodes.Items {
...@@ -3555,6 +3611,9 @@ func WaitForAllNodesHealthy(c clientset.Interface, timeout time.Duration) error ...@@ -3555,6 +3611,9 @@ func WaitForAllNodesHealthy(c clientset.Interface, timeout time.Duration) error
// It should be OK to list unschedulable Nodes here. // It should be OK to list unschedulable Nodes here.
nodes, err := c.Core().Nodes().List(metav1.ListOptions{ResourceVersion: "0"}) nodes, err := c.Core().Nodes().List(metav1.ListOptions{ResourceVersion: "0"})
if err != nil { if err != nil {
if IsRetryableAPIError(err) {
return false, nil
}
return false, err return false, err
} }
for _, node := range nodes.Items { for _, node := range nodes.Items {
...@@ -3897,7 +3956,10 @@ func WaitForIngressAddress(c clientset.Interface, ns, ingName string, timeout ti ...@@ -3897,7 +3956,10 @@ func WaitForIngressAddress(c clientset.Interface, ns, ingName string, timeout ti
ipOrNameList, err := getIngressAddress(c, ns, ingName) ipOrNameList, err := getIngressAddress(c, ns, ingName)
if err != nil || len(ipOrNameList) == 0 { if err != nil || len(ipOrNameList) == 0 {
Logf("Waiting for Ingress %v to acquire IP, error %v", ingName, err) Logf("Waiting for Ingress %v to acquire IP, error %v", ingName, err)
return false, nil if IsRetryableAPIError(err) {
return false, nil
}
return false, err
} }
address = ipOrNameList[0] address = ipOrNameList[0]
return true, nil return true, nil
...@@ -3974,7 +4036,13 @@ func GetNodePortURL(client clientset.Interface, ns, name string, svcPort int) (s ...@@ -3974,7 +4036,13 @@ func GetNodePortURL(client clientset.Interface, ns, name string, svcPort int) (s
nodes, err = client.Core().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{ nodes, err = client.Core().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{
"spec.unschedulable": "false", "spec.unschedulable": "false",
}.AsSelector().String()}) }.AsSelector().String()})
return err == nil, nil if err != nil {
if IsRetryableAPIError(err) {
return false, nil
}
return false, err
}
return true, nil
}) != nil { }) != nil {
return "", err return "", err
} }
...@@ -4792,3 +4860,7 @@ func DumpDebugInfo(c clientset.Interface, ns string) { ...@@ -4792,3 +4860,7 @@ func DumpDebugInfo(c clientset.Interface, ns string) {
Logf("\nLast 100 log lines of %v:\n%v", s.Name, l) Logf("\nLast 100 log lines of %v:\n%v", s.Name, l)
} }
} }
func IsRetryableAPIError(err error) bool {
return apierrs.IsTimeout(err) || apierrs.IsServerTimeout(err) || apierrs.IsTooManyRequests(err) || apierrs.IsInternalError(err)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment