integration: add retries to node authorizer tests

test/integration/auth/BUILD

@@ -58,6 +58,7 @@ go_test(
         "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/authentication/group:go_default_library",

test/integration/auth/node_test.go

@@ -20,8 +20,6 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
-	"path/filepath"
-	"runtime"
 	"testing"
 	"time"
 
@@ -29,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
 	"k8s.io/apiserver/pkg/authentication/token/tokenfile"
 	"k8s.io/apiserver/pkg/authentication/user"
@@ -149,28 +148,39 @@ func TestNodeAuthorizer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	getSecret := func(client clientset.Interface) error {
+	getSecret := func(client clientset.Interface) func() error {
+		return func() error {
 			_, err := client.Core().Secrets("ns").Get("mysecret", metav1.GetOptions{})
 			return err
 		}
-	getPVSecret := func(client clientset.Interface) error {
+	}
+	getPVSecret := func(client clientset.Interface) func() error {
+		return func() error {
 			_, err := client.Core().Secrets("ns").Get("mypvsecret", metav1.GetOptions{})
 			return err
 		}
-	getConfigMap := func(client clientset.Interface) error {
+	}
+	getConfigMap := func(client clientset.Interface) func() error {
+		return func() error {
 			_, err := client.Core().ConfigMaps("ns").Get("myconfigmap", metav1.GetOptions{})
 			return err
 		}
-	getPVC := func(client clientset.Interface) error {
+	}
+	getPVC := func(client clientset.Interface) func() error {
+		return func() error {
 			_, err := client.Core().PersistentVolumeClaims("ns").Get("mypvc", metav1.GetOptions{})
 			return err
 		}
-	getPV := func(client clientset.Interface) error {
+	}
+	getPV := func(client clientset.Interface) func() error {
+		return func() error {
 			_, err := client.Core().PersistentVolumes().Get("mypv", metav1.GetOptions{})
 			return err
 		}
+	}
 
-	createNode2NormalPod := func(client clientset.Interface) error {
+	createNode2NormalPod := func(client clientset.Interface) func() error {
+		return func() error {
 			_, err := client.Core().Pods("ns").Create(&api.Pod{
 				ObjectMeta: metav1.ObjectMeta{Name: "node2normalpod"},
 				Spec: api.PodSpec{
@@ -185,7 +195,9 @@ func TestNodeAuthorizer(t *testing.T) {
 			})
 			return err
 		}
-	updateNode2NormalPodStatus := func(client clientset.Interface) error {
+	}
+	updateNode2NormalPodStatus := func(client clientset.Interface) func() error {
+		return func() error {
 			startTime := metav1.NewTime(time.Now())
 			_, err := client.Core().Pods("ns").UpdateStatus(&api.Pod{
 				ObjectMeta: metav1.ObjectMeta{Name: "node2normalpod"},
@@ -193,12 +205,16 @@ func TestNodeAuthorizer(t *testing.T) {
 			})
 			return err
 		}
-	deleteNode2NormalPod := func(client clientset.Interface) error {
+	}
+	deleteNode2NormalPod := func(client clientset.Interface) func() error {
+		return func() error {
 			zero := int64(0)
 			return client.Core().Pods("ns").Delete("node2normalpod", &metav1.DeleteOptions{GracePeriodSeconds: &zero})
 		}
+	}
 
-	createNode2MirrorPod := func(client clientset.Interface) error {
+	createNode2MirrorPod := func(client clientset.Interface) func() error {
+		return func() error {
 			_, err := client.Core().Pods("ns").Create(&api.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:        "node2mirrorpod",
@@ -211,26 +227,36 @@ func TestNodeAuthorizer(t *testing.T) {
 			})
 			return err
 		}
-	deleteNode2MirrorPod := func(client clientset.Interface) error {
+	}
+	deleteNode2MirrorPod := func(client clientset.Interface) func() error {
+		return func() error {
 			zero := int64(0)
 			return client.Core().Pods("ns").Delete("node2mirrorpod", &metav1.DeleteOptions{GracePeriodSeconds: &zero})
 		}
+	}
 
-	createNode2 := func(client clientset.Interface) error {
+	createNode2 := func(client clientset.Interface) func() error {
+		return func() error {
 			_, err := client.Core().Nodes().Create(&api.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2"}})
 			return err
 		}
-	updateNode2Status := func(client clientset.Interface) error {
+	}
+	updateNode2Status := func(client clientset.Interface) func() error {
+		return func() error {
 			_, err := client.Core().Nodes().UpdateStatus(&api.Node{
 				ObjectMeta: metav1.ObjectMeta{Name: "node2"},
 				Status:     api.NodeStatus{},
 			})
 			return err
 		}
-	deleteNode2 := func(client clientset.Interface) error {
+	}
+	deleteNode2 := func(client clientset.Interface) func() error {
+		return func() error {
 			return client.Core().Nodes().Delete("node2", nil)
 		}
-	createNode2NormalPodEviction := func(client clientset.Interface) error {
+	}
+	createNode2NormalPodEviction := func(client clientset.Interface) func() error {
+		return func() error {
 			return client.Policy().Evictions("ns").Evict(&policy.Eviction{
 				TypeMeta: metav1.TypeMeta{
 					APIVersion: "policy/v1beta1",
@@ -242,7 +268,9 @@ func TestNodeAuthorizer(t *testing.T) {
 				},
 			})
 		}
-	createNode2MirrorPodEviction := func(client clientset.Interface) error {
+	}
+	createNode2MirrorPodEviction := func(client clientset.Interface) func() error {
+		return func() error {
 			return client.Policy().Evictions("ns").Evict(&policy.Eviction{
 				TypeMeta: metav1.TypeMeta{
 					APIVersion: "policy/v1beta1",
@@ -254,21 +282,26 @@ func TestNodeAuthorizer(t *testing.T) {
 				},
 			})
 		}
+	}
 
 	capacity := 50
-	updatePVCCapacity := func(client clientset.Interface) error {
+	updatePVCCapacity := func(client clientset.Interface) func() error {
+		return func() error {
 			capacity++
 			statusString := fmt.Sprintf("{\"status\": {\"capacity\": {\"storage\": \"%dG\"}}}", capacity)
 			patchBytes := []byte(statusString)
 			_, err := client.Core().PersistentVolumeClaims("ns").Patch("mypvc", types.StrategicMergePatchType, patchBytes, "status")
 			return err
 		}
+	}
 
-	updatePVCPhase := func(client clientset.Interface) error {
+	updatePVCPhase := func(client clientset.Interface) func() error {
+		return func() error {
 			patchBytes := []byte(`{"status":{"phase": "Bound"}}`)
 			_, err := client.Core().PersistentVolumeClaims("ns").Patch("mypvc", types.StrategicMergePatchType, patchBytes, "status")
 			return err
 		}
+	}
 
 	nodeanonClient := clientsetForToken(tokenNodeUnknown, clientConfig)
 	node1Client := clientsetForToken(tokenNode1, clientConfig)
@@ -386,23 +419,37 @@ func TestNodeAuthorizer(t *testing.T) {
 	expectForbidden(t, updatePVCPhase(node2Client))
 }
 
-func expectForbidden(t *testing.T, err error) {
-	if !errors.IsForbidden(err) {
-		_, file, line, _ := runtime.Caller(1)
-		t.Errorf("%s:%d: Expected forbidden error, got %v", filepath.Base(file), line, err)
+// expect executes a function a set number of times until it either returns the
+// expected error or executes too many times. It returns if the retries timed
+// out and the last error returned by the method.
+func expect(f func() error, wantErr func(error) bool) (timeout bool, lastErr error) {
+	err := wait.PollImmediate(time.Second, 30*time.Second, func() (bool, error) {
+		lastErr = f()
+		if wantErr(lastErr) {
+			return true, nil
 		}
+		return false, nil
+	})
+	return err == nil, lastErr
 }
 
-func expectNotFound(t *testing.T, err error) {
-	if !errors.IsNotFound(err) {
-		_, file, line, _ := runtime.Caller(1)
-		t.Errorf("%s:%d: Expected notfound error, got %v", filepath.Base(file), line, err)
+func expectForbidden(t *testing.T, f func() error) {
+	t.Helper()
+	if ok, err := expect(f, errors.IsForbidden); !ok {
+		t.Errorf("Expected forbidden error, got %v", err)
 	}
 }
 
-func expectAllowed(t *testing.T, err error) {
-	if err != nil {
-		_, file, line, _ := runtime.Caller(1)
-		t.Errorf("%s:%d: Expected no error, got %v", filepath.Base(file), line, err)
+func expectNotFound(t *testing.T, f func() error) {
+	t.Helper()
+	if ok, err := expect(f, errors.IsNotFound); !ok {
+		t.Errorf("Expected notfound error, got %v", err)
+	}
+}
+
+func expectAllowed(t *testing.T, f func() error) {
+	t.Helper()
+	if ok, err := expect(f, func(e error) bool { return e == nil }); !ok {
+		t.Errorf("Expected no error, got %v", err)
 	}
 }