Merge pull request #38255 from bprashanth/svc_cleanup

test/e2e/BUILD

@@ -57,7 +57,6 @@ go_library(
         "generated_clientset.go",
         "gke_local_ssd.go",
         "gke_node_pools.go",
-        "google_compute.go",
         "ha_master.go",
         "horizontal_pod_autoscaling.go",
         "ingress.go",

test/e2e/framework/BUILD

@@ -17,6 +17,7 @@ go_library(
         "exec_util.go",
         "federation_util.go",
         "framework.go",
+        "google_compute.go",
         "kubelet_stats.go",
         "log_size_monitoring.go",
         "metrics_util.go",
@@ -99,6 +100,7 @@ go_library(
         "//vendor:github.com/spf13/viper",
         "//vendor:golang.org/x/crypto/ssh",
         "//vendor:golang.org/x/net/websocket",
+        "//vendor:google.golang.org/api/googleapi",
         "//vendor:gopkg.in/yaml.v2",
         "//vendor:k8s.io/client-go/kubernetes",
         "//vendor:k8s.io/client-go/pkg/util/sets",

test/e2e/google_compute.go → test/e2e/framework/google_compute.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e
+package framework
 
 import (
 	"fmt"
@@ -26,13 +26,12 @@ import (
 	"github.com/golang/glog"
 
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
-	"k8s.io/kubernetes/test/e2e/framework"
 )
 
 // TODO: These should really just use the GCE API client library or at least use
 // better formatted output from the --format flag.
 
-func createGCEStaticIP(name string) (string, error) {
+func CreateGCEStaticIP(name string) (string, error) {
 	// gcloud compute --project "abshah-kubernetes-001" addresses create "test-static-ip" --region "us-central1"
 	// abshah@abhidesk:~/go/src/code.google.com/p/google-api-go-client/compute/v1$ gcloud compute --project "abshah-kubernetes-001" addresses create "test-static-ip" --region "us-central1"
 	// Created [https://www.googleapis.com/compute/v1/projects/abshah-kubernetes-001/regions/us-central1/addresses/test-static-ip].
@@ -41,14 +40,14 @@ func createGCEStaticIP(name string) (string, error) {
 
 	var outputBytes []byte
 	var err error
-	region, err := gce.GetGCERegion(framework.TestContext.CloudConfig.Zone)
+	region, err := gce.GetGCERegion(TestContext.CloudConfig.Zone)
 	if err != nil {
 		return "", fmt.Errorf("failed to convert zone to region: %v", err)
 	}
-	glog.Infof("Creating static IP with name %q in project %q in region %q", name, framework.TestContext.CloudConfig.ProjectID, region)
+	glog.Infof("Creating static IP with name %q in project %q in region %q", name, TestContext.CloudConfig.ProjectID, region)
 	for attempts := 0; attempts < 4; attempts++ {
 		outputBytes, err = exec.Command("gcloud", "compute", "addresses", "create",
-			name, "--project", framework.TestContext.CloudConfig.ProjectID,
+			name, "--project", TestContext.CloudConfig.ProjectID,
 			"--region", region, "-q").CombinedOutput()
 		if err == nil {
 			break
@@ -75,20 +74,20 @@ func createGCEStaticIP(name string) (string, error) {
 	}
 }
 
-func deleteGCEStaticIP(name string) error {
+func DeleteGCEStaticIP(name string) error {
 	// gcloud compute --project "abshah-kubernetes-001" addresses create "test-static-ip" --region "us-central1"
 	// abshah@abhidesk:~/go/src/code.google.com/p/google-api-go-client/compute/v1$ gcloud compute --project "abshah-kubernetes-001" addresses create "test-static-ip" --region "us-central1"
 	// Created [https://www.googleapis.com/compute/v1/projects/abshah-kubernetes-001/regions/us-central1/addresses/test-static-ip].
 	// NAME           REGION      ADDRESS       STATUS
 	// test-static-ip us-central1 104.197.143.7 RESERVED
 
-	region, err := gce.GetGCERegion(framework.TestContext.CloudConfig.Zone)
+	region, err := gce.GetGCERegion(TestContext.CloudConfig.Zone)
 	if err != nil {
 		return fmt.Errorf("failed to convert zone to region: %v", err)
 	}
-	glog.Infof("Deleting static IP with name %q in project %q in region %q", name, framework.TestContext.CloudConfig.ProjectID, region)
+	glog.Infof("Deleting static IP with name %q in project %q in region %q", name, TestContext.CloudConfig.ProjectID, region)
 	outputBytes, err := exec.Command("gcloud", "compute", "addresses", "delete",
-		name, "--project", framework.TestContext.CloudConfig.ProjectID,
+		name, "--project", TestContext.CloudConfig.ProjectID,
 		"--region", region, "-q").CombinedOutput()
 	if err != nil {
 		// Ditch the error, since the stderr in the output is what actually contains

test/e2e/framework/util.go

@@ -41,6 +41,7 @@ import (
 	"time"
 
 	"github.com/golang/glog"
+	"google.golang.org/api/googleapi"
 	"k8s.io/kubernetes/federation/client/clientset_generated/federation_release_1_5"
 	"k8s.io/kubernetes/pkg/api"
 	apierrs "k8s.io/kubernetes/pkg/api/errors"
@@ -5000,17 +5001,49 @@ func (p *E2ETestNodePreparer) CleanupNodes() error {
 	return encounteredError
 }
 
-func CleanupGCEResources(loadBalancerName string) (err error) {
+// CleanupGCEResources cleans up GCE Service Type=LoadBalancer resources with
+// the given name. The name is usually the UUID of the Service prefixed with an
+// alpha-numeric character ('a') to work around cloudprovider rules.
+func CleanupGCEResources(loadBalancerName string) (retErr error) {
 	gceCloud, ok := TestContext.CloudConfig.Provider.(*gcecloud.GCECloud)
 	if !ok {
 		return fmt.Errorf("failed to convert CloudConfig.Provider to GCECloud: %#v", TestContext.CloudConfig.Provider)
 	}
-	gceCloud.DeleteFirewall(loadBalancerName)
-	gceCloud.DeleteForwardingRule(loadBalancerName)
-	gceCloud.DeleteGlobalStaticIP(loadBalancerName)
-	hc, _ := gceCloud.GetHttpHealthCheck(loadBalancerName)
-	gceCloud.DeleteTargetPool(loadBalancerName, hc)
-	return nil
+	if err := gceCloud.DeleteFirewall(loadBalancerName); err != nil &&
+		!IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {
+		retErr = err
+	}
+	if err := gceCloud.DeleteForwardingRule(loadBalancerName); err != nil &&
+		!IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {
+		retErr = fmt.Errorf("%v\n%v", retErr, err)
+
+	}
+	if err := gceCloud.DeleteGlobalStaticIP(loadBalancerName); err != nil &&
+		!IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {
+		retErr = fmt.Errorf("%v\n%v", retErr, err)
+	}
+	// This function shells out to gcloud, so we can't compare for NotFound errors.
+	// TODO: Invoke cloudprovider method directly instead.
+	if err := DeleteGCEStaticIP(loadBalancerName); err != nil {
+		Logf("%v", err)
+	}
+	hc, getErr := gceCloud.GetHttpHealthCheck(loadBalancerName)
+	if getErr != nil && !IsGoogleAPIHTTPErrorCode(getErr, http.StatusNotFound) {
+		retErr = fmt.Errorf("%v\n%v", retErr, getErr)
+		return
+	}
+	if err := gceCloud.DeleteTargetPool(loadBalancerName, hc); err != nil &&
+		!IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {
+		retErr = fmt.Errorf("%v\n%v", retErr, err)
+	}
+	return
+}
+
+// IsHTTPErrorCode returns true if the error is a google api
+// error matching the corresponding HTTP error code.
+func IsGoogleAPIHTTPErrorCode(err error, code int) bool {
+	apiErr, ok := err.(*googleapi.Error)
+	return ok && apiErr.Code == code
 }
 
 // getMaster populates the externalIP, internalIP and hostname fields of the master.

test/e2e/service.go

@@ -580,12 +580,12 @@ var _ = framework.KubeDescribe("Services", func() {
 		if framework.ProviderIs("gce", "gke") {
 			By("creating a static load balancer IP")
 			staticIPName = fmt.Sprintf("e2e-external-lb-test-%s", framework.RunId)
-			requestedIP, err = createGCEStaticIP(staticIPName)
+			requestedIP, err = framework.CreateGCEStaticIP(staticIPName)
 			Expect(err).NotTo(HaveOccurred())
 			defer func() {
 				if staticIPName != "" {
 					// Release GCE static IP - this is not kube-managed and will not be automatically released.
-					if err := deleteGCEStaticIP(staticIPName); err != nil {
+					if err := framework.DeleteGCEStaticIP(staticIPName); err != nil {
 						framework.Logf("failed to release static IP %s: %v", staticIPName, err)
 					}
 				}
@@ -632,7 +632,7 @@ var _ = framework.KubeDescribe("Services", func() {
 			if staticIPName != "" {
 				// Deleting it after it is attached "demotes" it to an
 				// ephemeral IP, which can be auto-released.
-				if err := deleteGCEStaticIP(staticIPName); err != nil {
+				if err := framework.DeleteGCEStaticIP(staticIPName); err != nil {
 					framework.Failf("failed to release static IP %s: %v", staticIPName, err)
 				}
 				staticIPName = ""