Merge pull request #34444 from gmarek/refactor-sched-pred

test/e2e/common/networking.go

@@ -20,7 +20,6 @@ import (
 	. "github.com/onsi/ginkgo"
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/test/e2e/framework"
-	networking_util "k8s.io/kubernetes/test/utils"
 )
 
 var _ = framework.KubeDescribe("Networking", func() {
@@ -32,30 +31,30 @@ var _ = framework.KubeDescribe("Networking", func() {
 		// expect exactly one unique hostname. Each of these endpoints reports
 		// its own hostname.
 		It("should function for intra-pod communication: http [Conformance]", func() {
-			config := networking_util.NewCoreNetworkingTestConfig(f)
+			config := framework.NewCoreNetworkingTestConfig(f)
 			for _, endpointPod := range config.EndpointPods {
-				config.DialFromTestContainer("http", endpointPod.Status.PodIP, networking_util.EndpointHttpPort, config.MaxTries, 0, sets.NewString(endpointPod.Name))
+				config.DialFromTestContainer("http", endpointPod.Status.PodIP, framework.EndpointHttpPort, config.MaxTries, 0, sets.NewString(endpointPod.Name))
 			}
 		})
 
 		It("should function for intra-pod communication: udp [Conformance]", func() {
-			config := networking_util.NewCoreNetworkingTestConfig(f)
+			config := framework.NewCoreNetworkingTestConfig(f)
 			for _, endpointPod := range config.EndpointPods {
-				config.DialFromTestContainer("udp", endpointPod.Status.PodIP, networking_util.EndpointUdpPort, config.MaxTries, 0, sets.NewString(endpointPod.Name))
+				config.DialFromTestContainer("udp", endpointPod.Status.PodIP, framework.EndpointUdpPort, config.MaxTries, 0, sets.NewString(endpointPod.Name))
 			}
 		})
 
 		It("should function for node-pod communication: http [Conformance]", func() {
-			config := networking_util.NewCoreNetworkingTestConfig(f)
+			config := framework.NewCoreNetworkingTestConfig(f)
 			for _, endpointPod := range config.EndpointPods {
-				config.DialFromNode("http", endpointPod.Status.PodIP, networking_util.EndpointHttpPort, config.MaxTries, 0, sets.NewString(endpointPod.Name))
+				config.DialFromNode("http", endpointPod.Status.PodIP, framework.EndpointHttpPort, config.MaxTries, 0, sets.NewString(endpointPod.Name))
 			}
 		})
 
 		It("should function for node-pod communication: udp [Conformance]", func() {
-			config := networking_util.NewCoreNetworkingTestConfig(f)
+			config := framework.NewCoreNetworkingTestConfig(f)
 			for _, endpointPod := range config.EndpointPods {
-				config.DialFromNode("udp", endpointPod.Status.PodIP, networking_util.EndpointUdpPort, config.MaxTries, 0, sets.NewString(endpointPod.Name))
+				config.DialFromNode("udp", endpointPod.Status.PodIP, framework.EndpointUdpPort, config.MaxTries, 0, sets.NewString(endpointPod.Name))
 			}
 		})
 	})

test/e2e/framework/util.go

@@ -79,6 +79,7 @@ import (
 	utilyaml "k8s.io/kubernetes/pkg/util/yaml"
 	"k8s.io/kubernetes/pkg/version"
 	"k8s.io/kubernetes/pkg/watch"
+	testutil "k8s.io/kubernetes/test/utils"
 
 	"github.com/blang/semver"
 	"golang.org/x/crypto/ssh"
@@ -3022,64 +3023,25 @@ func WaitForAllNodesSchedulable(c *client.Client) error {
 	})
 }
 
-func AddOrUpdateLabelOnNode(c *client.Client, nodeName string, labelKey string, labelValue string) {
-	patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, labelKey, labelValue)
-	var err error
-	for attempt := 0; attempt < UpdateRetries; attempt++ {
-		err = c.Patch(api.MergePatchType).Resource("nodes").Name(nodeName).Body([]byte(patch)).Do().Error()
-		if err != nil {
-			if !apierrs.IsConflict(err) {
-				ExpectNoError(err)
-			} else {
-				Logf("Conflict when trying to add a label %v:%v to %v", labelKey, labelValue, nodeName)
-			}
-		} else {
-			break
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	ExpectNoError(err)
+func AddOrUpdateLabelOnNode(c clientset.Interface, nodeName string, labelKey, labelValue string) {
+	ExpectNoError(testutil.AddLabelsToNode(c, nodeName, map[string]string{labelKey: labelValue}))
 }
 
-func ExpectNodeHasLabel(c *client.Client, nodeName string, labelKey string, labelValue string) {
+func ExpectNodeHasLabel(c clientset.Interface, nodeName string, labelKey string, labelValue string) {
 	By("verifying the node has the label " + labelKey + " " + labelValue)
-	node, err := c.Nodes().Get(nodeName)
+	node, err := c.Core().Nodes().Get(nodeName)
 	ExpectNoError(err)
 	Expect(node.Labels[labelKey]).To(Equal(labelValue))
 }
 
 // RemoveLabelOffNode is for cleaning up labels temporarily added to node,
 // won't fail if target label doesn't exist or has been removed.
-func RemoveLabelOffNode(c *client.Client, nodeName string, labelKey string) {
+func RemoveLabelOffNode(c clientset.Interface, nodeName string, labelKey string) {
 	By("removing the label " + labelKey + " off the node " + nodeName)
-	var nodeUpdated *api.Node
-	var node *api.Node
-	var err error
-	for attempt := 0; attempt < UpdateRetries; attempt++ {
-		node, err = c.Nodes().Get(nodeName)
-		ExpectNoError(err)
-		if node.Labels == nil || len(node.Labels[labelKey]) == 0 {
-			return
-		}
-		delete(node.Labels, labelKey)
-		nodeUpdated, err = c.Nodes().Update(node)
-		if err != nil {
-			if !apierrs.IsConflict(err) {
-				ExpectNoError(err)
-			} else {
-				Logf("Conflict when trying to remove a label %v from %v", labelKey, nodeName)
-			}
-		} else {
-			break
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	ExpectNoError(err)
+	ExpectNoError(testutil.RemoveLabelOffNode(c, nodeName, []string{labelKey}))
 
 	By("verifying the node doesn't have the label " + labelKey)
-	if nodeUpdated.Labels != nil && len(nodeUpdated.Labels[labelKey]) != 0 {
-		Failf("Failed removing label " + labelKey + " of the node " + nodeName)
-	}
+	ExpectNoError(testutil.VerifyLabelsRemoved(c, nodeName, []string{labelKey}))
 }
 
 func AddOrUpdateTaintOnNode(c *client.Client, nodeName string, taint api.Taint) {

test/e2e/scheduler_predicates.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/errors"
 	"k8s.io/kubernetes/pkg/api/resource"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/util/uuid"
@@ -48,6 +49,7 @@ type pausePodConfig struct {
 
 var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 	var c *client.Client
+	var cs clientset.Interface
 	var nodeList *api.NodeList
 	var systemPodsNo int
 	var totalPodCapacity int64
@@ -67,6 +69,7 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 
 	BeforeEach(func() {
 		c = f.Client
+		cs = f.ClientSet
 		ns = f.Namespace.Name
 		nodeList = &api.NodeList{}
 
@@ -262,9 +265,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		k := fmt.Sprintf("kubernetes.io/e2e-%s", string(uuid.NewUUID()))
 		v := "42"
-		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		defer framework.RemoveLabelOffNode(c, nodeName, k)
+		framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v)
+		framework.ExpectNodeHasLabel(cs, nodeName, k, v)
+		defer framework.RemoveLabelOffNode(cs, nodeName, k)
 
 		By("Trying to relaunch the pod, now with labels.")
 		labelPodName := "with-labels"
@@ -333,9 +336,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		k := fmt.Sprintf("kubernetes.io/e2e-%s", string(uuid.NewUUID()))
 		v := "42"
-		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		defer framework.RemoveLabelOffNode(c, nodeName, k)
+		framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v)
+		framework.ExpectNodeHasLabel(cs, nodeName, k, v)
+		defer framework.RemoveLabelOffNode(cs, nodeName, k)
 
 		By("Trying to relaunch the pod, now with labels.")
 		labelPodName := "with-labels"
@@ -378,9 +381,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a label with fake az info on the found node.")
 		k := "kubernetes.io/e2e-az-name"
 		v := "e2e-az1"
-		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		defer framework.RemoveLabelOffNode(c, nodeName, k)
+		framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v)
+		framework.ExpectNodeHasLabel(cs, nodeName, k, v)
+		defer framework.RemoveLabelOffNode(cs, nodeName, k)
 
 		By("Trying to launch a pod that with NodeAffinity setting as embedded JSON string in the annotation value.")
 		pod := createPodWithNodeAffinity(f)
@@ -466,9 +469,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		k := "e2e.inter-pod-affinity.kubernetes.io/zone"
 		v := "china-e2etest"
-		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		defer framework.RemoveLabelOffNode(c, nodeName, k)
+		framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v)
+		framework.ExpectNodeHasLabel(cs, nodeName, k, v)
+		defer framework.RemoveLabelOffNode(cs, nodeName, k)
 
 		By("Trying to launch the pod, now with podAffinity.")
 		labelPodName := "with-podaffinity-" + string(uuid.NewUUID())
@@ -521,9 +524,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		k := "e2e.inter-pod-affinity.kubernetes.io/zone"
 		v := "china-e2etest"
 		for _, nodeName := range nodeNames {
-			framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-			framework.ExpectNodeHasLabel(c, nodeName, k, v)
-			defer framework.RemoveLabelOffNode(c, nodeName, k)
+			framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v)
+			framework.ExpectNodeHasLabel(cs, nodeName, k, v)
+			defer framework.RemoveLabelOffNode(cs, nodeName, k)
 		}
 
 		By("Trying to launch another pod on the first node with the service label.")
@@ -569,9 +572,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		k := "e2e.inter-pod-affinity.kubernetes.io/zone"
 		v := "kubernetes-e2e"
-		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		defer framework.RemoveLabelOffNode(c, nodeName, k)
+		framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v)
+		framework.ExpectNodeHasLabel(cs, nodeName, k, v)
+		defer framework.RemoveLabelOffNode(cs, nodeName, k)
 
 		By("Trying to launch the pod, now with multiple pod affinities with diff LabelOperators.")
 		labelPodName := "with-podaffinity-" + string(uuid.NewUUID())
@@ -620,9 +623,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		k := "e2e.inter-pod-affinity.kubernetes.io/zone"
 		v := "e2e-testing"
-		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		defer framework.RemoveLabelOffNode(c, nodeName, k)
+		framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v)
+		framework.ExpectNodeHasLabel(cs, nodeName, k, v)
+		defer framework.RemoveLabelOffNode(cs, nodeName, k)
 
 		By("Trying to launch the pod, now with Pod affinity and anti affinity.")
 		pod := createPodWithPodAffinity(f, k)
@@ -645,9 +648,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a label with fake az info on the found node.")
 		k := "e2e.inter-pod-affinity.kubernetes.io/zone"
 		v := "e2e-az1"
-		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		defer framework.RemoveLabelOffNode(c, nodeName, k)
+		framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v)
+		framework.ExpectNodeHasLabel(cs, nodeName, k, v)
+		defer framework.RemoveLabelOffNode(cs, nodeName, k)
 
 		By("Trying to launch a pod that with PodAffinity & PodAntiAffinity setting as embedded JSON string in the annotation value.")
 		pod := createPodWithPodAffinity(f, "kubernetes.io/hostname")
@@ -682,9 +685,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		labelKey := fmt.Sprintf("kubernetes.io/e2e-label-key-%s", string(uuid.NewUUID()))
 		labelValue := "testing-label-value"
-		framework.AddOrUpdateLabelOnNode(c, nodeName, labelKey, labelValue)
-		framework.ExpectNodeHasLabel(c, nodeName, labelKey, labelValue)
-		defer framework.RemoveLabelOffNode(c, nodeName, labelKey)
+		framework.AddOrUpdateLabelOnNode(cs, nodeName, labelKey, labelValue)
+		framework.ExpectNodeHasLabel(cs, nodeName, labelKey, labelValue)
+		defer framework.RemoveLabelOffNode(cs, nodeName, labelKey)
 
 		By("Trying to relaunch the pod, now with tolerations.")
 		tolerationPodName := "with-tolerations"
@@ -734,9 +737,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		labelKey := fmt.Sprintf("kubernetes.io/e2e-label-key-%s", string(uuid.NewUUID()))
 		labelValue := "testing-label-value"
-		framework.AddOrUpdateLabelOnNode(c, nodeName, labelKey, labelValue)
-		framework.ExpectNodeHasLabel(c, nodeName, labelKey, labelValue)
-		defer framework.RemoveLabelOffNode(c, nodeName, labelKey)
+		framework.AddOrUpdateLabelOnNode(cs, nodeName, labelKey, labelValue)
+		framework.ExpectNodeHasLabel(cs, nodeName, labelKey, labelValue)
+		defer framework.RemoveLabelOffNode(cs, nodeName, labelKey)
 
 		By("Trying to relaunch the pod, still no tolerations.")
 		podNameNoTolerations := "still-no-tolerations"

test/utils/density_utils.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api"
+	apierrs "k8s.io/kubernetes/pkg/api/errors"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
+
+	"github.com/golang/glog"
+)
+
+const (
+	retries = 5
+)
+
+func AddLabelsToNode(c clientset.Interface, nodeName string, labels map[string]string) error {
+	tokens := make([]string, 0, len(labels))
+	for k, v := range labels {
+		tokens = append(tokens, "\""+k+"\":\""+v+"\"")
+	}
+	labelString := "{" + strings.Join(tokens, ",") + "}"
+	patch := fmt.Sprintf(`{"metadata":{"labels":%v}}`, labelString)
+	var err error
+	for attempt := 0; attempt < retries; attempt++ {
+		_, err = c.Core().Nodes().Patch(nodeName, api.MergePatchType, []byte(patch))
+		if err != nil {
+			if !apierrs.IsConflict(err) {
+				return err
+			}
+		} else {
+			break
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return err
+}
+
+// RemoveLabelOffNode is for cleaning up labels temporarily added to node,
+// won't fail if target label doesn't exist or has been removed.
+func RemoveLabelOffNode(c clientset.Interface, nodeName string, labelKeys []string) error {
+	var node *api.Node
+	var err error
+	for attempt := 0; attempt < retries; attempt++ {
+		node, err = c.Core().Nodes().Get(nodeName)
+		if err != nil {
+			return err
+		}
+		if node.Labels == nil {
+			return nil
+		}
+		for _, labelKey := range labelKeys {
+			if node.Labels == nil || len(node.Labels[labelKey]) == 0 {
+				break
+			}
+			delete(node.Labels, labelKey)
+		}
+		_, err = c.Core().Nodes().Update(node)
+		if err != nil {
+			if !apierrs.IsConflict(err) {
+				return err
+			} else {
+				glog.V(2).Infof("Conflict when trying to remove a labels %v from %v", labelKeys, nodeName)
+			}
+		} else {
+			break
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return err
+}
+
+// VerifyLabelsRemoved checks if Node for given nodeName does not have any of labels from labelKeys.
+// Return non-nil error if it does.
+func VerifyLabelsRemoved(c clientset.Interface, nodeName string, labelKeys []string) error {
+	node, err := c.Core().Nodes().Get(nodeName)
+	if err != nil {
+		return err
+	}
+	for _, labelKey := range labelKeys {
+		if node.Labels != nil && len(node.Labels[labelKey]) != 0 {
+			return fmt.Errorf("Failed removing label " + labelKey + " of the node " + nodeName)
+		}
+	}
+	return nil
+}