GCE: Add "Network Tiers" as an Alpha feature for L4 load balancers

pkg/cloudprovider/providers/gce/gce_addresses.go

@@ -176,3 +176,15 @@ func (gce *GCECloud) GetBetaRegionAddressByIP(region, ipAddress string) (*comput
 	}
 	return nil, makeGoogleAPINotFoundError(fmt.Sprintf("Address with IP %q was not found in region %q", ipAddress, region))
 }
+
+// TODO(#51665): retire this function once Network Tiers becomes Beta in GCP.
+func (gce *GCECloud) getNetworkTierFromAddress(name, region string) (string, error) {
+	if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
+		return NetworkTierDefault.ToGCEValue(), nil
+	}
+	addr, err := gce.GetAlphaRegionAddress(name, region)
+	if err != nil {
+		return handleAlphaNetworkTierGetError(err)
+	}
+	return addr.NetworkTier, nil
+}

pkg/cloudprovider/providers/gce/gce_addresses_fakes.go

@@ -27,6 +27,8 @@ import (
 	compute "google.golang.org/api/compute/v1"
 )
 
+// test
+
 type FakeCloudAddressService struct {
 	count int
 	// reservedAddrs tracks usage of IP addresses
@@ -70,7 +72,16 @@ func (cas *FakeCloudAddressService) ReserveAlphaRegionAddress(addr *computealpha
 	}
 
 	if cas.reservedAddrs[addr.Address] {
-		return makeGoogleAPIError(http.StatusConflict, "IP in use")
+		msg := "IP in use"
+		// When the IP is already in use, this call returns an error code based
+		// on the type (internal vs external) of the address. This is to be
+		// consistent with actual GCE API.
+		switch lbScheme(addr.AddressType) {
+		case schemeExternal:
+			return makeGoogleAPIError(http.StatusBadRequest, msg)
+		default:
+			return makeGoogleAPIError(http.StatusConflict, msg)
+		}
 	}
 
 	if _, exists := cas.addrsByRegionAndName[region]; !exists {
@@ -133,6 +144,7 @@ func (cas *FakeCloudAddressService) DeleteRegionAddress(name, region string) err
 	if !exists {
 		return makeGoogleAPINotFoundError("")
 	}
+
 	delete(cas.reservedAddrs, addr.Address)
 	delete(cas.addrsByRegionAndName[region], name)
 	return nil
@@ -167,6 +179,14 @@ func (cas *FakeCloudAddressService) GetRegionAddressByIP(name, region string) (*
 	return nil, err
 }
 
+func (cas *FakeCloudAddressService) getNetworkTierFromAddress(name, region string) (string, error) {
+	addr, err := cas.GetAlphaRegionAddress(name, region)
+	if err != nil {
+		return "", err
+	}
+	return addr.NetworkTier, nil
+}
+
 func convertToV1Address(object gceObject) *compute.Address {
 	enc, err := object.MarshalJSON()
 	if err != nil {
@@ -188,6 +208,8 @@ func convertToAlphaAddress(object gceObject) *computealpha.Address {
 	if err := json.Unmarshal(enc, &addr); err != nil {
 		panic(fmt.Sprintf("Failed to convert GCE apiObject %v to alpha address: %v", object, err))
 	}
+	// Set the default values for the Alpha fields.
+	addr.NetworkTier = NetworkTierDefault.ToGCEValue()
 	return &addr
 }
 

pkg/cloudprovider/providers/gce/gce_alpha.go

@@ -22,15 +22,22 @@ import (
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 )
 
-// All known alpha features
-var knownAlphaFeatures = map[string]bool{
-	GCEDiskAlphaFeatureGate: true,
-}
-
 const (
+	// alpha: v1.8 (for Services)
+	//
+	// Allows Services backed by a GCP load balancer to choose what network
+	// tier to use. Currently supports "Standard" and "Premium" (default).
+	AlphaFeatureNetworkTiers = "NetworkTiers"
+
 	GCEDiskAlphaFeatureGate = "GCEDiskAlphaAPI"
 )
 
+// All known alpha features
+var knownAlphaFeatures = map[string]bool{
+	AlphaFeatureNetworkTiers: true,
+	GCEDiskAlphaFeatureGate:  true,
+}
+
 type AlphaFeatureGate struct {
 	features map[string]bool
 }

pkg/cloudprovider/providers/gce/gce_forwardingrule.go

@@ -141,3 +141,15 @@ func (gce *GCECloud) DeleteRegionForwardingRule(name, region string) error {
 
 	return gce.waitForRegionOp(op, region, mc)
 }
+
+// TODO(#51665): retire this function once Network Tiers becomes Beta in GCP.
+func (gce *GCECloud) getNetworkTierFromForwardingRule(name, region string) (string, error) {
+	if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
+		return NetworkTierDefault.ToGCEValue(), nil
+	}
+	fwdRule, err := gce.GetAlphaRegionForwardingRule(name, region)
+	if err != nil {
+		return handleAlphaNetworkTierGetError(err)
+	}
+	return fwdRule.NetworkTier, nil
+}

pkg/cloudprovider/providers/gce/gce_forwardingrule_fakes.go

@@ -102,6 +102,14 @@ func (f *FakeCloudForwardingRuleService) GetRegionForwardingRule(name, region st
 	return nil, err
 }
 
+func (f *FakeCloudForwardingRuleService) getNetworkTierFromForwardingRule(name, region string) (string, error) {
+	fwdRule, err := f.GetAlphaRegionForwardingRule(name, region)
+	if err != nil {
+		return "", err
+	}
+	return fwdRule.NetworkTier, nil
+}
+
 func convertToV1ForwardingRule(object gceObject) *compute.ForwardingRule {
 	enc, err := object.MarshalJSON()
 	if err != nil {
@@ -123,5 +131,8 @@ func convertToAlphaForwardingRule(object gceObject) *computealpha.ForwardingRule
 	if err := json.Unmarshal(enc, &fwdRule); err != nil {
 		panic(fmt.Sprintf("Failed to convert GCE apiObject %v to alpha fwdRuleess: %v", object, err))
 	}
+	// Set the default values for the Alpha fields.
+	fwdRule.NetworkTier = NetworkTierDefault.ToGCEValue()
+
 	return &fwdRule
 }

pkg/cloudprovider/providers/gce/gce_interfaces.go

@@ -22,6 +22,8 @@ import (
 	compute "google.golang.org/api/compute/v1"
 )
 
+// These interfaces are added for testability.
+
 // CloudAddressService is an interface for managing addresses
 type CloudAddressService interface {
 	ReserveRegionAddress(address *compute.Address, region string) error
@@ -38,6 +40,9 @@ type CloudAddressService interface {
 	ReserveBetaRegionAddress(address *computebeta.Address, region string) error
 	GetBetaRegionAddress(name string, region string) (*computebeta.Address, error)
 	GetBetaRegionAddressByIP(region, ipAddress string) (*computebeta.Address, error)
+
+	// TODO(#51665): Remove this once the Network Tiers becomes Alpha in GCP.
+	getNetworkTierFromAddress(name, region string) (string, error)
 }
 
 // CloudForwardingRuleService is an interface for managing forwarding rules.
@@ -50,4 +55,7 @@ type CloudForwardingRuleService interface {
 	// Alpha API.
 	GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error)
 	CreateAlphaRegionForwardingRule(rule *computealpha.ForwardingRule, region string) error
+
+	// Needed for the Alpha "Network Tiers" feature.
+	getNetworkTierFromForwardingRule(name, region string) (string, error)
 }

pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go

@@ -21,8 +21,10 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-
+	"github.com/stretchr/testify/require"
 	computealpha "google.golang.org/api/compute/v0.alpha"
+
+	"k8s.io/api/core/v1"
 )
 
 func TestEnsureStaticIP(t *testing.T) {
@@ -32,19 +34,54 @@ func TestEnsureStaticIP(t *testing.T) {
 	region := "us-central1"
 
 	// First ensure call
-	ip, existed, err := ensureStaticIP(fcas, ipName, serviceName, region, "")
+	ip, existed, err := ensureStaticIP(fcas, ipName, serviceName, region, "", NetworkTierDefault)
 	if err != nil || existed || ip == "" {
 		t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, "") = %v, %v, %v; want valid ip, false, nil`, fcas, ipName, serviceName, region, ip, existed, err)
 	}
 
 	// Second ensure call
 	var ipPrime string
-	ipPrime, existed, err = ensureStaticIP(fcas, ipName, serviceName, region, ip)
+	ipPrime, existed, err = ensureStaticIP(fcas, ipName, serviceName, region, ip, NetworkTierDefault)
 	if err != nil || !existed || ip != ipPrime {
 		t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, %v) = %v, %v, %v; want %v, true, nil`, fcas, ipName, serviceName, region, ip, ipPrime, existed, err, ip)
 	}
 }
 
+func TestEnsureStaticIPWithTier(t *testing.T) {
+	s := NewFakeCloudAddressService()
+	serviceName := ""
+	region := "us-east1"
+
+	for desc, tc := range map[string]struct {
+		name     string
+		netTier  NetworkTier
+		expected string
+	}{
+		"Premium (default)": {
+			name:     "foo-1",
+			netTier:  NetworkTierPremium,
+			expected: "PREMIUM",
+		},
+		"Standard": {
+			name:     "foo-2",
+			netTier:  NetworkTierStandard,
+			expected: "STANDARD",
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			ip, existed, err := ensureStaticIP(s, tc.name, serviceName, region, "", tc.netTier)
+			assert.NoError(t, err)
+			assert.False(t, existed)
+			assert.NotEqual(t, "", ip)
+			// Get the Address from the fake address service and verify that the tier
+			// is set correctly.
+			alphaAddr, err := s.GetAlphaRegionAddress(tc.name, region)
+			require.NoError(t, err)
+			assert.Equal(t, tc.expected, alphaAddr.NetworkTier)
+		})
+	}
+}
+
 func TestVerifyRequestedIP(t *testing.T) {
 	region := "test-region"
 	lbRef := "test-lb"
@@ -53,35 +90,150 @@ func TestVerifyRequestedIP(t *testing.T) {
 	for desc, tc := range map[string]struct {
 		requestedIP     string
 		fwdRuleIP       string
+		netTier         NetworkTier
 		addrList        []*computealpha.Address
 		expectErr       bool
 		expectUserOwned bool
 	}{
 		"requested IP exists": {
 			requestedIP:     "1.1.1.1",
-			addrList:        []*computealpha.Address{{Name: "foo", Address: "1.1.1.1"}},
+			netTier:         NetworkTierPremium,
+			addrList:        []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
 			expectErr:       false,
 			expectUserOwned: true,
 		},
 		"requested IP is not static, but is in use by the fwd rule": {
 			requestedIP: "1.1.1.1",
 			fwdRuleIP:   "1.1.1.1",
+			netTier:     NetworkTierPremium,
 			expectErr:   false,
 		},
 		"requested IP is not static and is not used by the fwd rule": {
 			requestedIP: "1.1.1.1",
 			fwdRuleIP:   "2.2.2.2",
+			netTier:     NetworkTierPremium,
 			expectErr:   true,
 		},
 		"no requested IP": {
+			netTier:   NetworkTierPremium,
 			expectErr: false,
 		},
+		"requested IP exists, but network tier does not match": {
+			requestedIP: "1.1.1.1",
+			netTier:     NetworkTierStandard,
+			addrList:    []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
+			expectErr:   true,
+		},
 	} {
 		t.Run(desc, func(t *testing.T) {
 			s.SetRegionalAddresses(region, tc.addrList)
-			isUserOwnedIP, err := verifyUserRequestedIP(s, region, tc.requestedIP, tc.fwdRuleIP, lbRef)
+			isUserOwnedIP, err := verifyUserRequestedIP(s, region, tc.requestedIP, tc.fwdRuleIP, lbRef, tc.netTier)
 			assert.Equal(t, tc.expectErr, err != nil, fmt.Sprintf("err: %v", err))
-			assert.Equal(t, tc.expectUserOwned, isUserOwnedIP, desc)
+			assert.Equal(t, tc.expectUserOwned, isUserOwnedIP)
+		})
+	}
+}
+
+func TestCreateForwardingRuleWithTier(t *testing.T) {
+	s := NewFakeCloudForwardingRuleService()
+	// Common variables among the tests.
+	ports := []v1.ServicePort{{Name: "foo", Protocol: v1.ProtocolTCP, Port: int32(123)}}
+	region := "test-region"
+	target := "test-target-pool"
+	svcName := "foo-svc"
+
+	for desc, tc := range map[string]struct {
+		netTier      NetworkTier
+		expectedRule *computealpha.ForwardingRule
+	}{
+		"Premium tier": {
+			netTier: NetworkTierPremium,
+			expectedRule: &computealpha.ForwardingRule{
+				Name:        "lb-1",
+				Description: `{"kubernetes.io/service-name":"foo-svc"}`,
+				IPAddress:   "1.1.1.1",
+				IPProtocol:  "TCP",
+				PortRange:   "123-123",
+				Target:      target,
+				NetworkTier: "PREMIUM",
+			},
+		},
+		"Standard tier": {
+			netTier: NetworkTierStandard,
+			expectedRule: &computealpha.ForwardingRule{
+				Name:        "lb-2",
+				Description: `{"kubernetes.io/service-name":"foo-svc"}`,
+				IPAddress:   "2.2.2.2",
+				IPProtocol:  "TCP",
+				PortRange:   "123-123",
+				Target:      target,
+				NetworkTier: "STANDARD",
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			lbName := tc.expectedRule.Name
+			ipAddr := tc.expectedRule.IPAddress
+
+			err := createForwardingRule(s, lbName, svcName, region, ipAddr, target, ports, tc.netTier)
+			assert.NoError(t, err)
+
+			alphaRule, err := s.GetAlphaRegionForwardingRule(lbName, region)
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expectedRule, alphaRule)
+		})
+	}
+}
+
+func TestDeleteAddressWithWrongTier(t *testing.T) {
+	region := "test-region"
+	lbRef := "test-lb"
+	s := NewFakeCloudAddressService()
+
+	for desc, tc := range map[string]struct {
+		addrName     string
+		netTier      NetworkTier
+		addrList     []*computealpha.Address
+		expectDelete bool
+	}{
+		"Network tiers (premium) match; do nothing": {
+			addrName: "foo1",
+			netTier:  NetworkTierPremium,
+			addrList: []*computealpha.Address{{Name: "foo1", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
+		},
+		"Network tiers (standard) match; do nothing": {
+			addrName: "foo2",
+			netTier:  NetworkTierStandard,
+			addrList: []*computealpha.Address{{Name: "foo2", Address: "1.1.1.2", NetworkTier: "STANDARD"}},
+		},
+		"Wrong network tier (standard); delete address": {
+			addrName:     "foo3",
+			netTier:      NetworkTierPremium,
+			addrList:     []*computealpha.Address{{Name: "foo3", Address: "1.1.1.3", NetworkTier: "STANDARD"}},
+			expectDelete: true,
+		},
+		"Wrong network tier (preimium); delete address": {
+			addrName:     "foo4",
+			netTier:      NetworkTierStandard,
+			addrList:     []*computealpha.Address{{Name: "foo4", Address: "1.1.1.4", NetworkTier: "PREMIUM"}},
+			expectDelete: true,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			s.SetRegionalAddresses(region, tc.addrList)
+			// Sanity check to ensure we inject the right address.
+			_, err := s.GetRegionAddress(tc.addrName, region)
+			require.NoError(t, err)
+
+			err = deleteAddressWithWrongTier(s, region, tc.addrName, lbRef, tc.netTier)
+			assert.NoError(t, err)
+			// Check whether the address still exists.
+			_, err = s.GetRegionAddress(tc.addrName, region)
+			if tc.expectDelete {
+				assert.True(t, isNotFound(err))
+			} else {
+				assert.NoError(t, err)
+			}
 		})
 	}
 }

pkg/cloudprovider/providers/gce/gce_loadbalancer_naming.go

@@ -84,6 +84,11 @@ func makeBackendServiceDescription(nm types.NamespacedName, shared bool) string
 
 // External Load Balancer
 
+// makeServiceDescription is used to generate descriptions for forwarding rules and addresses.
+func makeServiceDescription(serviceName string) string {
+	return fmt.Sprintf(`{"kubernetes.io/service-name":"%s"}`, serviceName)
+}
+
 // makeNodesHealthCheckName returns name of the health check resource used by
 // the GCE load balancers (l4) for performing health checks on nodes.
 func makeNodesHealthCheckName(clusterID string) string {

pkg/cloudprovider/providers/gce/gce_util.go

@@ -157,3 +157,19 @@ func makeGoogleAPINotFoundError(message string) error {
 func makeGoogleAPIError(code int, message string) error {
 	return &googleapi.Error{Code: code, Message: message}
 }
+
+func isForbidden(err error) bool {
+	return isHTTPErrorCode(err, http.StatusForbidden)
+}
+
+// TODO(#51665): Remove this once Network Tiers becomes Beta in GCP.
+func handleAlphaNetworkTierGetError(err error) (string, error) {
+	if isForbidden(err) {
+		// Network tier is still an Alpha feature in GCP, and not every project
+		// is whitelisted to access the API. If we cannot access the API, just
+		// assume the tier is premium.
+		return NetworkTierDefault.ToGCEValue(), nil
+	}
+	// Can't get the network tier, just return an error.
+	return "", err
+}