Merge pull request #43003 from deads2k/server-05-discovery

pkg/master/BUILD

@@ -85,6 +85,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",

pkg/master/master.go

@@ -26,6 +26,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
 	"k8s.io/apiserver/pkg/registry/generic"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/healthz"
@@ -163,9 +164,9 @@ func (c *Config) Complete() completedConfig {
 		c.APIServerServiceIP = apiServerServiceIP
 	}
 
-	discoveryAddresses := genericapiserver.DefaultDiscoveryAddresses{DefaultAddress: c.GenericConfig.ExternalAddress}
-	discoveryAddresses.DiscoveryCIDRRules = append(discoveryAddresses.DiscoveryCIDRRules,
-		genericapiserver.DiscoveryCIDRRule{IPRange: c.ServiceIPRange, Address: net.JoinHostPort(c.APIServerServiceIP.String(), strconv.Itoa(c.APIServerServicePort))})
+	discoveryAddresses := discovery.DefaultAddresses{DefaultAddress: c.GenericConfig.ExternalAddress}
+	discoveryAddresses.CIDRRules = append(discoveryAddresses.CIDRRules,
+		discovery.CIDRRule{IPRange: c.ServiceIPRange, Address: net.JoinHostPort(c.APIServerServiceIP.String(), strconv.Itoa(c.APIServerServicePort))})
 	c.GenericConfig.DiscoveryAddresses = discoveryAddresses
 
 	if c.ServiceNodePortRange.Size == 0 {
@@ -249,7 +250,7 @@ func (c completedConfig) New() (*Master, error) {
 		autoscalingrest.RESTStorageProvider{},
 		batchrest.RESTStorageProvider{},
 		certificatesrest.RESTStorageProvider{},
-		extensionsrest.RESTStorageProvider{ResourceInterface: thirdparty.NewThirdPartyResourceServer(s, c.StorageFactory)},
+		extensionsrest.RESTStorageProvider{ResourceInterface: thirdparty.NewThirdPartyResourceServer(s, s.DiscoveryGroupManager, c.StorageFactory)},
 		policyrest.RESTStorageProvider{},
 		rbacrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorizer},
 		settingsrest.RESTStorageProvider{},

pkg/master/thirdparty/BUILD

@@ -33,7 +33,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/endpoints/handlers:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/registry/rest:go_default_library",

pkg/master/thirdparty/thirdparty.go

@@ -28,7 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	genericapi "k8s.io/apiserver/pkg/endpoints"
-	genericapihandlers "k8s.io/apiserver/pkg/endpoints/handlers"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/registry/rest"
@@ -53,11 +53,13 @@ func (d dynamicLister) ListAPIResources() []metav1.APIResource {
 	return d.m.getExistingThirdPartyResources(d.path)
 }
 
-var _ genericapihandlers.APIResourceLister = &dynamicLister{}
+var _ discovery.APIResourceLister = &dynamicLister{}
 
 type ThirdPartyResourceServer struct {
 	genericAPIServer *genericapiserver.GenericAPIServer
 
+	availableGroupManager discovery.GroupManager
+
 	deleteCollectionWorkers int
 
 	// storage for third party objects
@@ -71,10 +73,11 @@ type ThirdPartyResourceServer struct {
 	disableThirdPartyControllerForTesting bool
 }
 
-func NewThirdPartyResourceServer(genericAPIServer *genericapiserver.GenericAPIServer, storageFactory serverstorgage.StorageFactory) *ThirdPartyResourceServer {
+func NewThirdPartyResourceServer(genericAPIServer *genericapiserver.GenericAPIServer, availableGroupManager discovery.GroupManager, storageFactory serverstorgage.StorageFactory) *ThirdPartyResourceServer {
 	ret := &ThirdPartyResourceServer{
 		genericAPIServer:      genericAPIServer,
 		thirdPartyResources:   map[string]*thirdPartyEntry{},
+		availableGroupManager: availableGroupManager,
 	}
 
 	var err error
@@ -133,7 +136,7 @@ func (m *ThirdPartyResourceServer) removeThirdPartyStorage(path, resource string
 	delete(entry.storage, resource)
 	if len(entry.storage) == 0 {
 		delete(m.thirdPartyResources, path)
-		m.genericAPIServer.RemoveAPIGroupForDiscovery(extensionsrest.GetThirdPartyGroupName(path))
+		m.availableGroupManager.RemoveGroup(extensionsrest.GetThirdPartyGroupName(path))
 	} else {
 		m.thirdPartyResources[path] = entry
 	}
@@ -236,7 +239,7 @@ func (m *ThirdPartyResourceServer) addThirdPartyResourceStorage(path, resource s
 	}
 	entry.storage[resource] = storage
 	if !found {
-		m.genericAPIServer.AddAPIGroupForDiscovery(apiGroup)
+		m.availableGroupManager.AddGroup(apiGroup)
 	}
 }
 
@@ -284,7 +287,7 @@ func (m *ThirdPartyResourceServer) InstallThirdPartyResource(rsrc *extensions.Th
 	if err := thirdparty.InstallREST(m.genericAPIServer.HandlerContainer.Container); err != nil {
 		glog.Errorf("Unable to setup thirdparty api: %v", err)
 	}
-	m.genericAPIServer.HandlerContainer.Add(genericapi.NewGroupWebService(api.Codecs, path, apiGroup))
+	m.genericAPIServer.HandlerContainer.Add(discovery.NewAPIGroupHandler(api.Codecs, apiGroup).WebService())
 
 	m.addThirdPartyResourceStorage(path, plural.Resource, thirdparty.Storage[plural.Resource].(*thirdpartyresourcedatastore.REST), apiGroup)
 	api.Registry.AddThirdPartyAPIGroupVersions(schema.GroupVersion{Group: group, Version: rsrc.Versions[0].Name})

staging/src/k8s.io/apiserver/pkg/endpoints/BUILD

@@ -55,7 +55,6 @@ go_library(
     name = "go_default_library",
     srcs = [
         "apiserver.go",
-        "discovery.go",
         "doc.go",
         "groupversion.go",
         "installer.go",
@@ -72,9 +71,9 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints/handlers:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints/handlers/negotiation:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints/metrics:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/registry/rest:go_default_library",

staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "addresses_test.go",
+        "root_test.go",
+    ],
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "addresses.go",
+        "group.go",
+        "legacy.go",
+        "root.go",
+        "util.go",
+        "version.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//vendor/github.com/emicklei/go-restful:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/handlers/negotiation:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters:go_default_library",
+    ],
+)

staging/src/k8s.io/apiserver/pkg/server/discovery.go → staging/src/k8s.io/apiserver/pkg/endpoints/discovery/addresses.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package server
+package discovery
 
 import (
 	"net"
@@ -22,22 +22,22 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-type DiscoveryAddresses interface {
+type Addresses interface {
 	ServerAddressByClientCIDRs(net.IP) []metav1.ServerAddressByClientCIDR
 }
 
-// DefaultDiscoveryAddresses is a default implementation of DiscoveryAddresses that will work in most cases
-type DefaultDiscoveryAddresses struct {
-	// DiscoveryCIDRRules is a list of CIDRs and Addresses to use if a client is in the range
-	DiscoveryCIDRRules []DiscoveryCIDRRule
+// DefaultAddresses is a default implementation of Addresses that will work in most cases
+type DefaultAddresses struct {
+	// CIDRRules is a list of CIDRs and Addresses to use if a client is in the range
+	CIDRRules []CIDRRule
 
 	// DefaultAddress is the address (hostname or IP and port) that should be used in
 	// if no CIDR matches more specifically.
 	DefaultAddress string
 }
 
-// DiscoveryCIDRRule is a rule for adding an alternate path to the master based on matching CIDR
-type DiscoveryCIDRRule struct {
+// CIDRRule is a rule for adding an alternate path to the master based on matching CIDR
+type CIDRRule struct {
 	IPRange net.IPNet
 
 	// Address is the address (hostname or IP and port) that should be used in
@@ -45,7 +45,7 @@ type DiscoveryCIDRRule struct {
 	Address string
 }
 
-func (d DefaultDiscoveryAddresses) ServerAddressByClientCIDRs(clientIP net.IP) []metav1.ServerAddressByClientCIDR {
+func (d DefaultAddresses) ServerAddressByClientCIDRs(clientIP net.IP) []metav1.ServerAddressByClientCIDR {
 	addressCIDRMap := []metav1.ServerAddressByClientCIDR{
 		{
 			ClientCIDR:    "0.0.0.0/0",
@@ -53,13 +53,13 @@ func (d DefaultDiscoveryAddresses) ServerAddressByClientCIDRs(clientIP net.IP) [
 		},
 	}
 
-	for _, rule := range d.DiscoveryCIDRRules {
+	for _, rule := range d.CIDRRules {
 		addressCIDRMap = append(addressCIDRMap, rule.ServerAddressByClientCIDRs(clientIP)...)
 	}
 	return addressCIDRMap
 }
 
-func (d DiscoveryCIDRRule) ServerAddressByClientCIDRs(clientIP net.IP) []metav1.ServerAddressByClientCIDR {
+func (d CIDRRule) ServerAddressByClientCIDRs(clientIP net.IP) []metav1.ServerAddressByClientCIDR {
 	addressCIDRMap := []metav1.ServerAddressByClientCIDR{}
 
 	if d.IPRange.Contains(clientIP) {

staging/src/k8s.io/apiserver/pkg/endpoints/discovery/addresses_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package discovery
+
+import (
+	"net"
+	"net/http"
+	"reflect"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+)
+
+func TestGetServerAddressByClientCIDRs(t *testing.T) {
+	publicAddressCIDRMap := []metav1.ServerAddressByClientCIDR{
+		{
+			ClientCIDR:    "0.0.0.0/0",
+			ServerAddress: "ExternalAddress",
+		},
+	}
+	internalAddressCIDRMap := []metav1.ServerAddressByClientCIDR{
+		publicAddressCIDRMap[0],
+		{
+			ClientCIDR:    "10.0.0.0/24",
+			ServerAddress: "serviceIP",
+		},
+	}
+	internalIP := "10.0.0.1"
+	publicIP := "1.1.1.1"
+	testCases := []struct {
+		Request     http.Request
+		ExpectedMap []metav1.ServerAddressByClientCIDR
+	}{
+		{
+			Request:     http.Request{},
+			ExpectedMap: publicAddressCIDRMap,
+		},
+		{
+			Request: http.Request{
+				Header: map[string][]string{
+					"X-Real-Ip": {internalIP},
+				},
+			},
+			ExpectedMap: internalAddressCIDRMap,
+		},
+		{
+			Request: http.Request{
+				Header: map[string][]string{
+					"X-Real-Ip": {publicIP},
+				},
+			},
+			ExpectedMap: publicAddressCIDRMap,
+		},
+		{
+			Request: http.Request{
+				Header: map[string][]string{
+					"X-Forwarded-For": {internalIP},
+				},
+			},
+			ExpectedMap: internalAddressCIDRMap,
+		},
+		{
+			Request: http.Request{
+				Header: map[string][]string{
+					"X-Forwarded-For": {publicIP},
+				},
+			},
+			ExpectedMap: publicAddressCIDRMap,
+		},
+
+		{
+			Request: http.Request{
+				RemoteAddr: internalIP,
+			},
+			ExpectedMap: internalAddressCIDRMap,
+		},
+		{
+			Request: http.Request{
+				RemoteAddr: publicIP,
+			},
+			ExpectedMap: publicAddressCIDRMap,
+		},
+		{
+			Request: http.Request{
+				RemoteAddr: "invalidIP",
+			},
+			ExpectedMap: publicAddressCIDRMap,
+		},
+	}
+
+	_, ipRange, _ := net.ParseCIDR("10.0.0.0/24")
+	discoveryAddresses := DefaultAddresses{DefaultAddress: "ExternalAddress"}
+	discoveryAddresses.CIDRRules = append(discoveryAddresses.CIDRRules,
+		CIDRRule{IPRange: *ipRange, Address: "serviceIP"})
+
+	for i, test := range testCases {
+		if a, e := discoveryAddresses.ServerAddressByClientCIDRs(utilnet.GetClientIP(&test.Request)), test.ExpectedMap; reflect.DeepEqual(e, a) != true {
+			t.Fatalf("test case %d failed. expected: %v, actual: %v", i+1, e, a)
+		}
+	}
+}

staging/src/k8s.io/apiserver/pkg/endpoints/discovery/group.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package discovery
+
+import (
+	"net/http"
+
+	"github.com/emicklei/go-restful"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+)
+
+// apiGroupHandler creates a webservice serving the supported versions, preferred version, and name
+// of a group. E.g., such a web service will be registered at /apis/extensions.
+type apiGroupHandler struct {
+	serializer runtime.NegotiatedSerializer
+
+	group metav1.APIGroup
+}
+
+func NewAPIGroupHandler(serializer runtime.NegotiatedSerializer, group metav1.APIGroup) *apiGroupHandler {
+	if keepUnversioned(group.Name) {
+		// Because in release 1.1, /apis/extensions returns response with empty
+		// APIVersion, we use stripVersionNegotiatedSerializer to keep the
+		// response backwards compatible.
+		serializer = stripVersionNegotiatedSerializer{serializer}
+	}
+
+	return &apiGroupHandler{
+		serializer: serializer,
+		group:      group,
+	}
+}
+
+func (s *apiGroupHandler) WebService() *restful.WebService {
+	mediaTypes, _ := negotiation.MediaTypesForSerializer(s.serializer)
+	ws := new(restful.WebService)
+	ws.Path(APIGroupPrefix + "/" + s.group.Name)
+	ws.Doc("get information of a group")
+	ws.Route(ws.GET("/").To(s.handle).
+		Doc("get information of a group").
+		Operation("getAPIGroup").
+		Produces(mediaTypes...).
+		Consumes(mediaTypes...).
+		Writes(metav1.APIGroup{}))
+	return ws
+}
+
+// handle returns a handler which will return the api.GroupAndVersion of the group.
+func (s *apiGroupHandler) handle(req *restful.Request, resp *restful.Response) {
+	responsewriters.WriteObjectNegotiated(s.serializer, schema.GroupVersion{}, resp.ResponseWriter, req.Request, http.StatusOK, &s.group)
+}

staging/src/k8s.io/apiserver/pkg/endpoints/discovery/legacy.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package discovery
+
+import (
+	"net/http"
+
+	"github.com/emicklei/go-restful"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+)
+
+// legacyRootAPIHandler creates a webservice serving api group discovery.
+type legacyRootAPIHandler struct {
+	// addresses is used to build cluster IPs for discovery.
+	addresses   Addresses
+	apiPrefix   string
+	serializer  runtime.NegotiatedSerializer
+	apiVersions []string
+}
+
+func NewLegacyRootAPIHandler(addresses Addresses, serializer runtime.NegotiatedSerializer, apiPrefix string, apiVersions []string) *legacyRootAPIHandler {
+	// Because in release 1.1, /apis returns response with empty APIVersion, we
+	// use stripVersionNegotiatedSerializer to keep the response backwards
+	// compatible.
+	serializer = stripVersionNegotiatedSerializer{serializer}
+
+	return &legacyRootAPIHandler{
+		addresses:   addresses,
+		apiPrefix:   apiPrefix,
+		serializer:  serializer,
+		apiVersions: apiVersions,
+	}
+}
+
+// AddApiWebService adds a service to return the supported api versions at the legacy /api.
+func (s *legacyRootAPIHandler) WebService() *restful.WebService {
+	mediaTypes, _ := negotiation.MediaTypesForSerializer(s.serializer)
+	ws := new(restful.WebService)
+	ws.Path(s.apiPrefix)
+	ws.Doc("get available API versions")
+	ws.Route(ws.GET("/").To(s.handle).
+		Doc("get available API versions").
+		Operation("getAPIVersions").
+		Produces(mediaTypes...).
+		Consumes(mediaTypes...).
+		Writes(metav1.APIVersions{}))
+	return ws
+}
+
+func (s *legacyRootAPIHandler) handle(req *restful.Request, resp *restful.Response) {
+	clientIP := utilnet.GetClientIP(req.Request)
+	apiVersions := &metav1.APIVersions{
+		ServerAddressByClientCIDRs: s.addresses.ServerAddressByClientCIDRs(clientIP),
+		Versions:                   s.apiVersions,
+	}
+
+	responsewriters.WriteObjectNegotiated(s.serializer, schema.GroupVersion{}, resp.ResponseWriter, req.Request, http.StatusOK, apiVersions)
+}

staging/src/k8s.io/apiserver/pkg/endpoints/discovery/root.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package discovery
+
+import (
+	"net/http"
+	"sync"
+
+	"github.com/emicklei/go-restful"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+)
+
+// GroupManager is an interface that allows dynamic mutation of the existing webservice to handle
+// API groups being added or removed.
+type GroupManager interface {
+	AddGroup(apiGroup metav1.APIGroup)
+	RemoveGroup(groupName string)
+
+	WebService() *restful.WebService
+}
+
+// rootAPIsHandler creates a webservice serving api group discovery.
+// The list of APIGroups may change while the server is running because additional resources
+// are registered or removed.  It is not safe to cache the values.
+type rootAPIsHandler struct {
+	// addresses is used to build cluster IPs for discovery.
+	addresses Addresses
+
+	serializer runtime.NegotiatedSerializer
+
+	// Map storing information about all groups to be exposed in discovery response.
+	// The map is from name to the group.
+	lock      sync.RWMutex
+	apiGroups map[string]metav1.APIGroup
+	// apiGroupNames preserves insertion order
+	apiGroupNames []string
+}
+
+func NewRootAPIsHandler(addresses Addresses, serializer runtime.NegotiatedSerializer) *rootAPIsHandler {
+	// Because in release 1.1, /apis returns response with empty APIVersion, we
+	// use stripVersionNegotiatedSerializer to keep the response backwards
+	// compatible.
+	serializer = stripVersionNegotiatedSerializer{serializer}
+
+	return &rootAPIsHandler{
+		addresses:  addresses,
+		serializer: serializer,
+		apiGroups:  map[string]metav1.APIGroup{},
+	}
+}
+
+func (s *rootAPIsHandler) AddGroup(apiGroup metav1.APIGroup) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	_, alreadyExists := s.apiGroups[apiGroup.Name]
+
+	s.apiGroups[apiGroup.Name] = apiGroup
+	if !alreadyExists {
+		s.apiGroupNames = append(s.apiGroupNames, apiGroup.Name)
+	}
+}
+
+func (s *rootAPIsHandler) RemoveGroup(groupName string) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	delete(s.apiGroups, groupName)
+	for i := range s.apiGroupNames {
+		if s.apiGroupNames[i] == groupName {
+			s.apiGroupNames = append(s.apiGroupNames[:i], s.apiGroupNames[i+1:]...)
+			break
+		}
+	}
+}
+
+func (s *rootAPIsHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	orderedGroups := []metav1.APIGroup{}
+	for _, groupName := range s.apiGroupNames {
+		orderedGroups = append(orderedGroups, s.apiGroups[groupName])
+	}
+
+	clientIP := utilnet.GetClientIP(req)
+	serverCIDR := s.addresses.ServerAddressByClientCIDRs(clientIP)
+	groups := make([]metav1.APIGroup, len(orderedGroups))
+	for i := range orderedGroups {
+		groups[i] = orderedGroups[i]
+		groups[i].ServerAddressByClientCIDRs = serverCIDR
+	}
+
+	responsewriters.WriteObjectNegotiated(s.serializer, schema.GroupVersion{}, resp, req, http.StatusOK, &metav1.APIGroupList{Groups: groups})
+}
+
+func (s *rootAPIsHandler) restfulHandle(req *restful.Request, resp *restful.Response) {
+	s.ServeHTTP(resp.ResponseWriter, req.Request)
+}
+
+// WebService returns a webservice serving api group discovery.
+// Note: during the server runtime apiGroups might change.
+func (s *rootAPIsHandler) WebService() *restful.WebService {
+	mediaTypes, _ := negotiation.MediaTypesForSerializer(s.serializer)
+	ws := new(restful.WebService)
+	ws.Path(APIGroupPrefix)
+	ws.Doc("get available API versions")
+	ws.Route(ws.GET("/").To(s.restfulHandle).
+		Doc("get available API versions").
+		Operation("getAPIVersions").
+		Produces(mediaTypes...).
+		Consumes(mediaTypes...).
+		Writes(metav1.APIGroupList{}))
+	return ws
+}

staging/src/k8s.io/apiserver/pkg/endpoints/discovery/root_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package discovery
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"k8s.io/apimachinery/pkg/apimachinery/announced"
+	"k8s.io/apimachinery/pkg/apimachinery/registered"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+)
+
+var (
+	groupFactoryRegistry = make(announced.APIGroupFactoryRegistry)
+	registry             = registered.NewOrDie("")
+	scheme               = runtime.NewScheme()
+	codecs               = serializer.NewCodecFactory(scheme)
+)
+
+func init() {
+	// Register Unversioned types under their own special group
+	scheme.AddUnversionedTypes(schema.GroupVersion{Group: "", Version: "v1"},
+		&metav1.Status{},
+		&metav1.APIVersions{},
+		&metav1.APIGroupList{},
+		&metav1.APIGroup{},
+		&metav1.APIResourceList{},
+	)
+}
+
+func decodeResponse(t *testing.T, resp *http.Response, obj interface{}) error {
+	defer resp.Body.Close()
+
+	data, err := ioutil.ReadAll(resp.Body)
+	t.Log(string(data))
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(data, obj); err != nil {
+		return err
+	}
+	return nil
+}
+
+func getGroupList(t *testing.T, server *httptest.Server) (*metav1.APIGroupList, error) {
+	resp, err := http.Get(server.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected server response, expected %d, actual: %d", http.StatusOK, resp.StatusCode)
+	}
+
+	groupList := metav1.APIGroupList{}
+	err = decodeResponse(t, resp, &groupList)
+	return &groupList, err
+}
+
+func TestDiscoveryAtAPIS(t *testing.T) {
+	handler := NewRootAPIsHandler(DefaultAddresses{DefaultAddress: "192.168.1.1"}, codecs)
+
+	server := httptest.NewServer(handler)
+	groupList, err := getGroupList(t, server)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	assert.Equal(t, 0, len(groupList.Groups))
+
+	// Add a Group.
+	extensionsGroupName := "extensions"
+	extensionsVersions := []metav1.GroupVersionForDiscovery{
+		{
+			GroupVersion: extensionsGroupName + "/v1",
+			Version:      "v1",
+		},
+	}
+	extensionsPreferredVersion := metav1.GroupVersionForDiscovery{
+		GroupVersion: extensionsGroupName + "/preferred",
+		Version:      "preferred",
+	}
+	handler.AddGroup(metav1.APIGroup{
+		Name:             extensionsGroupName,
+		Versions:         extensionsVersions,
+		PreferredVersion: extensionsPreferredVersion,
+	})
+
+	groupList, err = getGroupList(t, server)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	assert.Equal(t, 1, len(groupList.Groups))
+	groupListGroup := groupList.Groups[0]
+	assert.Equal(t, extensionsGroupName, groupListGroup.Name)
+	assert.Equal(t, extensionsVersions, groupListGroup.Versions)
+	assert.Equal(t, extensionsPreferredVersion, groupListGroup.PreferredVersion)
+	assert.Equal(t, handler.addresses.ServerAddressByClientCIDRs(utilnet.GetClientIP(&http.Request{})), groupListGroup.ServerAddressByClientCIDRs)
+
+	// Remove the group.
+	handler.RemoveGroup(extensionsGroupName)
+	groupList, err = getGroupList(t, server)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	assert.Equal(t, 0, len(groupList.Groups))
+}
+
+func TestDiscoveryOrdering(t *testing.T) {
+	handler := NewRootAPIsHandler(DefaultAddresses{DefaultAddress: "192.168.1.1"}, codecs)
+
+	server := httptest.NewServer(handler)
+	groupList, err := getGroupList(t, server)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	assert.Equal(t, 0, len(groupList.Groups))
+
+	// Register three groups
+	handler.AddGroup(metav1.APIGroup{Name: "x"})
+	handler.AddGroup(metav1.APIGroup{Name: "y"})
+	handler.AddGroup(metav1.APIGroup{Name: "z"})
+	// Register three additional groups that come earlier alphabetically
+	handler.AddGroup(metav1.APIGroup{Name: "a"})
+	handler.AddGroup(metav1.APIGroup{Name: "b"})
+	handler.AddGroup(metav1.APIGroup{Name: "c"})
+	// Make sure re-adding doesn't double-register or make a group lose its place
+	handler.AddGroup(metav1.APIGroup{Name: "x"})
+
+	groupList, err = getGroupList(t, server)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	assert.Equal(t, 6, len(groupList.Groups))
+	assert.Equal(t, "x", groupList.Groups[0].Name)
+	assert.Equal(t, "y", groupList.Groups[1].Name)
+	assert.Equal(t, "z", groupList.Groups[2].Name)
+	assert.Equal(t, "a", groupList.Groups[3].Name)
+	assert.Equal(t, "b", groupList.Groups[4].Name)
+	assert.Equal(t, "c", groupList.Groups[5].Name)
+
+	// Remove a group.
+	handler.RemoveGroup("a")
+	groupList, err = getGroupList(t, server)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	assert.Equal(t, 5, len(groupList.Groups))
+
+	// Re-adding should move to the end.
+	handler.AddGroup(metav1.APIGroup{Name: "a"})
+	groupList, err = getGroupList(t, server)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	assert.Equal(t, 6, len(groupList.Groups))
+	assert.Equal(t, "x", groupList.Groups[0].Name)
+	assert.Equal(t, "y", groupList.Groups[1].Name)
+	assert.Equal(t, "z", groupList.Groups[2].Name)
+	assert.Equal(t, "b", groupList.Groups[3].Name)
+	assert.Equal(t, "c", groupList.Groups[4].Name)
+	assert.Equal(t, "a", groupList.Groups[5].Name)
+}

staging/src/k8s.io/apiserver/pkg/endpoints/discovery.go → staging/src/k8s.io/apiserver/pkg/endpoints/discovery/util.go

 /*
-Copyright 2014 The Kubernetes Authors.
+Copyright 2017 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,44 +14,20 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package endpoints
+package discovery
 
 import (
 	"bytes"
 	"fmt"
 	"io"
-	"net/http"
 
-	"github.com/emicklei/go-restful"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers"
-	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
-	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 )
 
-// AddApiWebService adds a service to return the supported api versions at the legacy /api.
-func AddApiWebService(s runtime.NegotiatedSerializer, container *restful.Container, apiPrefix string, getAPIVersionsFunc func(req *restful.Request) *metav1.APIVersions) {
-	// TODO: InstallREST should register each version automatically
+const APIGroupPrefix = "/apis"
 
-	// Because in release 1.1, /api returns response with empty APIVersion, we
-	// use StripVersionNegotiatedSerializer to keep the response backwards
-	// compatible.
-	mediaTypes, _ := negotiation.MediaTypesForSerializer(s)
-	ss := stripVersionNegotiatedSerializer{s}
-	versionHandler := APIVersionHandler(ss, getAPIVersionsFunc)
-	ws := new(restful.WebService)
-	ws.Path(apiPrefix)
-	ws.Doc("get available API versions")
-	ws.Route(ws.GET("/").To(versionHandler).
-		Doc("get available API versions").
-		Operation("getAPIVersions").
-		Produces(mediaTypes...).
-		Consumes(mediaTypes...).
-		Writes(metav1.APIVersions{}))
-	container.Add(ws)
+func keepUnversioned(group string) bool {
+	return group == "" || group == "extensions"
 }
 
 // stripVersionEncoder strips APIVersion field from the encoding output. It's
@@ -95,78 +71,3 @@ func (n stripVersionNegotiatedSerializer) EncoderForVersion(encoder runtime.Enco
 	versioned := n.NegotiatedSerializer.EncoderForVersion(encoder, gv)
 	return stripVersionEncoder{versioned, serializer}
 }
-
-func keepUnversioned(group string) bool {
-	return group == "" || group == "extensions"
-}
-
-// NewApisWebService returns a webservice serving the available api version under /apis.
-func NewApisWebService(s runtime.NegotiatedSerializer, apiPrefix string, f func(req *restful.Request) []metav1.APIGroup) *restful.WebService {
-	// Because in release 1.1, /apis returns response with empty APIVersion, we
-	// use StripVersionNegotiatedSerializer to keep the response backwards
-	// compatible.
-	ss := stripVersionNegotiatedSerializer{s}
-	mediaTypes, _ := negotiation.MediaTypesForSerializer(s)
-	rootAPIHandler := handlers.RootAPIHandler(ss, f)
-	ws := new(restful.WebService)
-	ws.Path(apiPrefix)
-	ws.Doc("get available API versions")
-	ws.Route(ws.GET("/").To(rootAPIHandler).
-		Doc("get available API versions").
-		Operation("getAPIVersions").
-		Produces(mediaTypes...).
-		Consumes(mediaTypes...).
-		Writes(metav1.APIGroupList{}))
-	return ws
-}
-
-// NewGroupWebService returns a webservice serving the supported versions, preferred version, and name
-// of a group. E.g., such a web service will be registered at /apis/extensions.
-func NewGroupWebService(s runtime.NegotiatedSerializer, path string, group metav1.APIGroup) *restful.WebService {
-	ss := s
-	if keepUnversioned(group.Name) {
-		// Because in release 1.1, /apis/extensions returns response with empty
-		// APIVersion, we use StripVersionNegotiatedSerializer to keep the
-		// response backwards compatible.
-		ss = stripVersionNegotiatedSerializer{s}
-	}
-	mediaTypes, _ := negotiation.MediaTypesForSerializer(s)
-	groupHandler := handlers.GroupHandler(ss, group)
-	ws := new(restful.WebService)
-	ws.Path(path)
-	ws.Doc("get information of a group")
-	ws.Route(ws.GET("/").To(groupHandler).
-		Doc("get information of a group").
-		Operation("getAPIGroup").
-		Produces(mediaTypes...).
-		Consumes(mediaTypes...).
-		Writes(metav1.APIGroup{}))
-	return ws
-}
-
-// Adds a service to return the supported resources, E.g., a such web service
-// will be registered at /apis/extensions/v1.
-func AddSupportedResourcesWebService(s runtime.NegotiatedSerializer, ws *restful.WebService, groupVersion schema.GroupVersion, lister handlers.APIResourceLister) {
-	ss := s
-	if keepUnversioned(groupVersion.Group) {
-		// Because in release 1.1, /apis/extensions/v1beta1 returns response
-		// with empty APIVersion, we use StripVersionNegotiatedSerializer to
-		// keep the response backwards compatible.
-		ss = stripVersionNegotiatedSerializer{s}
-	}
-	mediaTypes, _ := negotiation.MediaTypesForSerializer(s)
-	resourceHandler := handlers.SupportedResourcesHandler(ss, groupVersion, lister)
-	ws.Route(ws.GET("/").To(resourceHandler).
-		Doc("get available resources").
-		Operation("getAPIResources").
-		Produces(mediaTypes...).
-		Consumes(mediaTypes...).
-		Writes(metav1.APIResourceList{}))
-}
-
-// APIVersionHandler returns a handler which will list the provided versions as available.
-func APIVersionHandler(s runtime.NegotiatedSerializer, getAPIVersionsFunc func(req *restful.Request) *metav1.APIVersions) restful.RouteFunction {
-	return func(req *restful.Request, resp *restful.Response) {
-		responsewriters.WriteObjectNegotiated(s, schema.GroupVersion{}, resp.ResponseWriter, req.Request, http.StatusOK, getAPIVersionsFunc(req))
-	}
-}

staging/src/k8s.io/apiserver/pkg/endpoints/discovery/version.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package discovery
+
+import (
+	"net/http"
+
+	"github.com/emicklei/go-restful"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+)
+
+type APIResourceLister interface {
+	ListAPIResources() []metav1.APIResource
+}
+
+// apiVersionHandler creates a webservice serving the supported resources for the version
+// E.g., such a web service will be registered at /apis/extensions/v1beta1.
+type apiVersionHandler struct {
+	serializer runtime.NegotiatedSerializer
+
+	groupVersion      schema.GroupVersion
+	apiResourceLister APIResourceLister
+}
+
+func NewAPIVersionHandler(serializer runtime.NegotiatedSerializer, groupVersion schema.GroupVersion, apiResourceLister APIResourceLister) *apiVersionHandler {
+	if keepUnversioned(groupVersion.Group) {
+		// Because in release 1.1, /apis/extensions returns response with empty
+		// APIVersion, we use stripVersionNegotiatedSerializer to keep the
+		// response backwards compatible.
+		serializer = stripVersionNegotiatedSerializer{serializer}
+	}
+
+	return &apiVersionHandler{
+		serializer:        serializer,
+		groupVersion:      groupVersion,
+		apiResourceLister: apiResourceLister,
+	}
+}
+
+func (s *apiVersionHandler) AddToWebService(ws *restful.WebService) {
+	mediaTypes, _ := negotiation.MediaTypesForSerializer(s.serializer)
+	ws.Route(ws.GET("/").To(s.handle).
+		Doc("get available resources").
+		Operation("getAPIResources").
+		Produces(mediaTypes...).
+		Consumes(mediaTypes...).
+		Writes(metav1.APIResourceList{}))
+}
+
+// handle returns a handler which will return the api.VersionAndVersion of the group.
+func (s *apiVersionHandler) handle(req *restful.Request, resp *restful.Response) {
+	responsewriters.WriteObjectNegotiated(s.serializer, schema.GroupVersion{}, resp.ResponseWriter, req.Request, http.StatusOK,
+		&metav1.APIResourceList{GroupVersion: s.groupVersion.String(), APIResources: s.apiResourceLister.ListAPIResources()})
+}

staging/src/k8s.io/apiserver/pkg/endpoints/groupversion.go

@@ -30,7 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apiserver/pkg/admission"
-	"k8s.io/apiserver/pkg/endpoints/handlers"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
 )
@@ -86,7 +86,7 @@ type APIGroupVersion struct {
 
 	// ResourceLister is an interface that knows how to list resources
 	// for this API Group.
-	ResourceLister handlers.APIResourceLister
+	ResourceLister discovery.APIResourceLister
 }
 
 // InstallREST registers the REST handlers (storage, watch, proxy and redirect) into a restful Container.
@@ -100,7 +100,8 @@ func (g *APIGroupVersion) InstallREST(container *restful.Container) error {
 	if lister == nil {
 		lister = staticLister{apiResources}
 	}
-	AddSupportedResourcesWebService(g.Serializer, ws, g.GroupVersion, lister)
+	versionDiscoveryHandler := discovery.NewAPIVersionHandler(g.Serializer, g.GroupVersion, lister)
+	versionDiscoveryHandler.AddToWebService(ws)
 	container.Add(ws)
 	return utilerrors.NewAggregate(registrationErrors)
 }
@@ -128,7 +129,8 @@ func (g *APIGroupVersion) UpdateREST(container *restful.Container) error {
 	if lister == nil {
 		lister = staticLister{apiResources}
 	}
-	AddSupportedResourcesWebService(g.Serializer, ws, g.GroupVersion, lister)
+	versionDiscoveryHandler := discovery.NewAPIVersionHandler(g.Serializer, g.GroupVersion, lister)
+	versionDiscoveryHandler.AddToWebService(ws)
 	return utilerrors.NewAggregate(registrationErrors)
 }
 
@@ -152,4 +154,4 @@ func (s staticLister) ListAPIResources() []metav1.APIResource {
 	return s.list
 }
 
-var _ handlers.APIResourceLister = &staticLister{}
+var _ discovery.APIResourceLister = &staticLister{}

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD

@@ -35,7 +35,6 @@ go_test(
 go_library(
     name = "go_default_library",
     srcs = [
-        "discovery.go",
         "doc.go",
         "namer.go",
         "patch.go",
@@ -45,7 +44,6 @@ go_library(
     ],
     tags = ["automanaged"],
     deps = [
-        "//vendor/github.com/emicklei/go-restful:go_default_library",
         "//vendor/github.com/evanphx/json-patch:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/golang.org/x/net/websocket:go_default_library",

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/discovery.go

-/*
-Copyright 2014 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package handlers
-
-import (
-	"net/http"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
-
-	"github.com/emicklei/go-restful"
-)
-
-type APIResourceLister interface {
-	ListAPIResources() []metav1.APIResource
-}
-
-// RootAPIHandler returns a handler which will list the provided groups and versions as available.
-func RootAPIHandler(s runtime.NegotiatedSerializer, f func(req *restful.Request) []metav1.APIGroup) restful.RouteFunction {
-	return func(req *restful.Request, resp *restful.Response) {
-		responsewriters.WriteObjectNegotiated(s, schema.GroupVersion{}, resp.ResponseWriter, req.Request, http.StatusOK, &metav1.APIGroupList{Groups: f(req)})
-	}
-}
-
-// GroupHandler returns a handler which will return the api.GroupAndVersion of
-// the group.
-func GroupHandler(s runtime.NegotiatedSerializer, group metav1.APIGroup) restful.RouteFunction {
-	return func(req *restful.Request, resp *restful.Response) {
-		responsewriters.WriteObjectNegotiated(s, schema.GroupVersion{}, resp.ResponseWriter, req.Request, http.StatusOK, &group)
-	}
-}
-
-// SupportedResourcesHandler returns a handler which will list the provided resources as available.
-func SupportedResourcesHandler(s runtime.NegotiatedSerializer, groupVersion schema.GroupVersion, lister APIResourceLister) restful.RouteFunction {
-	return func(req *restful.Request, resp *restful.Response) {
-		responsewriters.WriteObjectNegotiated(s, schema.GroupVersion{}, resp.ResponseWriter, req.Request, http.StatusOK, &metav1.APIResourceList{GroupVersion: groupVersion.String(), APIResources: lister.ListAPIResources()})
-	}
-}

staging/src/k8s.io/apiserver/pkg/server/BUILD

@@ -25,13 +25,13 @@ go_test(
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/apis/example:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/apis/example/v1:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/registry/rest:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
@@ -46,7 +46,6 @@ go_library(
     srcs = [
         "config.go",
         "config_selfclient.go",
-        "discovery.go",
         "doc.go",
         "genericapiserver.go",
         "healthz.go",
@@ -56,7 +55,6 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//vendor/github.com/coreos/go-systemd/daemon:go_default_library",
-        "//vendor/github.com/emicklei/go-restful:go_default_library",
         "//vendor/github.com/emicklei/go-restful/swagger:go_default_library",
         "//vendor/github.com/go-openapi/spec:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
@@ -69,7 +67,6 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
@@ -84,6 +81,7 @@ go_library(
         "//vendor/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/authorization/union:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints/filters:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -33,7 +33,6 @@ import (
 	"github.com/go-openapi/spec"
 	"github.com/pborman/uuid"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	openapicommon "k8s.io/apimachinery/pkg/openapi"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -47,6 +46,7 @@ import (
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	authorizerunion "k8s.io/apiserver/pkg/authorization/union"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
@@ -122,7 +122,7 @@ type Config struct {
 	BuildHandlerChainFunc func(apiHandler http.Handler, c *Config) (secure http.Handler)
 	// DiscoveryAddresses is used to build the IPs pass to discovery.  If nil, the ExternalAddress is
 	// always reported
-	DiscoveryAddresses DiscoveryAddresses
+	DiscoveryAddresses discovery.Addresses
 	// The default set of healthz checks. There might be more added via AddHealthzChecks dynamically.
 	HealthzChecks []healthz.HealthzChecker
 	// LegacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests
@@ -321,7 +321,7 @@ func (c *Config) Complete() completedConfig {
 		}
 	}
 	if c.DiscoveryAddresses == nil {
-		c.DiscoveryAddresses = DefaultDiscoveryAddresses{DefaultAddress: c.ExternalAddress}
+		c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress}
 	}
 
 	// If the loopbackclientconfig is specified AND it has a token for use against the API server
@@ -408,8 +408,6 @@ func (c completedConfig) constructServer() (*GenericAPIServer, error) {
 		SecureServingInfo: c.SecureServingInfo,
 		ExternalAddress:   c.ExternalAddress,
 
-		apiGroupsForDiscovery: map[string]metav1.APIGroup{},
-
 		HandlerContainer:   handlerContainer,
 		FallThroughHandler: c.FallThroughHandler,
 
@@ -422,6 +420,8 @@ func (c completedConfig) constructServer() (*GenericAPIServer, error) {
 		disabledPostStartHooks: c.DisabledPostStartHooks,
 
 		healthzChecks: c.HealthzChecks,
+
+		DiscoveryGroupManager: discovery.NewRootAPIsHandler(c.DiscoveryAddresses, c.Serializer),
 	}
 
 	return s, nil
@@ -529,7 +529,7 @@ func installAPI(s *GenericAPIServer, c *Config, delegate http.Handler) {
 	routes.Version{Version: c.Version}.Install(s.HandlerContainer)
 
 	if c.EnableDiscovery {
-		s.HandlerContainer.Add(s.DynamicApisDiscovery())
+		s.HandlerContainer.Add(s.DiscoveryGroupManager.WebService())
 	}
 }
 

staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -25,7 +25,6 @@ import (
 	"time"
 
 	systemd "github.com/coreos/go-systemd/daemon"
-	"github.com/emicklei/go-restful"
 	"github.com/emicklei/go-restful/swagger"
 	"github.com/golang/glog"
 
@@ -36,10 +35,10 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
-	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/admission"
 	genericapi "k8s.io/apiserver/pkg/endpoints"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
 	"k8s.io/apiserver/pkg/server/healthz"
@@ -84,7 +83,7 @@ type APIGroupInfo struct {
 // GenericAPIServer contains state for a Kubernetes cluster api server.
 type GenericAPIServer struct {
 	// discoveryAddresses is used to build cluster IPs for discovery.
-	discoveryAddresses DiscoveryAddresses
+	discoveryAddresses discovery.Addresses
 
 	// LoopbackClientConfig is a config for a privileged loopback connection to the API server
 	LoopbackClientConfig *restclient.Config
@@ -130,12 +129,8 @@ type GenericAPIServer struct {
 	// listedPathProvider is a lister which provides the set of paths to show at /
 	listedPathProvider routes.ListedPathProvider
 
-	// Map storing information about all groups to be exposed in discovery response.
-	// The map is from name to the group.
-	// The slice preserves group name insertion order.
-	apiGroupsForDiscoveryLock sync.RWMutex
-	apiGroupsForDiscovery     map[string]metav1.APIGroup
-	apiGroupNamesForDiscovery []string
+	// DiscoveryGroupManager serves /apis
+	DiscoveryGroupManager discovery.GroupManager
 
 	// Enable swagger and/or OpenAPI if these configs are non-nil.
 	swaggerConfig *swagger.Config
@@ -334,15 +329,7 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo
 	}
 	// Install the version handler.
 	// Add a handler at /<apiPrefix> to enumerate the supported api versions.
-	genericapi.AddApiWebService(s.Serializer, s.HandlerContainer.Container, apiPrefix, func(req *restful.Request) *metav1.APIVersions {
-		clientIP := utilnet.GetClientIP(req.Request)
-
-		apiVersionsForDiscovery := metav1.APIVersions{
-			ServerAddressByClientCIDRs: s.discoveryAddresses.ServerAddressByClientCIDRs(clientIP),
-			Versions:                   apiVersions,
-		}
-		return &apiVersionsForDiscovery
-	})
+	s.HandlerContainer.Add(discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix, apiVersions).WebService())
 	return nil
 }
 
@@ -385,43 +372,12 @@ func (s *GenericAPIServer) InstallAPIGroup(apiGroupInfo *APIGroupInfo) error {
 		PreferredVersion: preferedVersionForDiscovery,
 	}
 
-	s.AddAPIGroupForDiscovery(apiGroup)
-	s.HandlerContainer.Add(genericapi.NewGroupWebService(s.Serializer, APIGroupPrefix+"/"+apiGroup.Name, apiGroup))
+	s.DiscoveryGroupManager.AddGroup(apiGroup)
+	s.HandlerContainer.Add(discovery.NewAPIGroupHandler(s.Serializer, apiGroup).WebService())
 
 	return nil
 }
 
-// AddAPIGroupForDiscovery adds the specified group to the list served to discovery queries.
-// Groups are listed in the order they are added.
-func (s *GenericAPIServer) AddAPIGroupForDiscovery(apiGroup metav1.APIGroup) {
-	s.apiGroupsForDiscoveryLock.Lock()
-	defer s.apiGroupsForDiscoveryLock.Unlock()
-
-	// Insert the group into the ordered list if it is not already present
-	if _, exists := s.apiGroupsForDiscovery[apiGroup.Name]; !exists {
-		s.apiGroupNamesForDiscovery = append(s.apiGroupNamesForDiscovery, apiGroup.Name)
-	}
-
-	s.apiGroupsForDiscovery[apiGroup.Name] = apiGroup
-}
-
-func (s *GenericAPIServer) RemoveAPIGroupForDiscovery(groupName string) {
-	s.apiGroupsForDiscoveryLock.Lock()
-	defer s.apiGroupsForDiscoveryLock.Unlock()
-
-	// Remove the group from the ordered list
-	// https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
-	newOrder := s.apiGroupNamesForDiscovery[:0]
-	for _, orderedGroupName := range s.apiGroupNamesForDiscovery {
-		if orderedGroupName != groupName {
-			newOrder = append(newOrder, orderedGroupName)
-		}
-	}
-	s.apiGroupNamesForDiscovery = newOrder
-
-	delete(s.apiGroupsForDiscovery, groupName)
-}
-
 func (s *GenericAPIServer) getAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion, apiPrefix string) *genericapi.APIGroupVersion {
 	storage := make(map[string]rest.Storage)
 	for k, v := range apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version] {
@@ -456,31 +412,6 @@ func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupV
 	}
 }
 
-// DynamicApisDiscovery returns a webservice serving api group discovery.
-// Note: during the server runtime apiGroupsForDiscovery might change.
-func (s *GenericAPIServer) DynamicApisDiscovery() *restful.WebService {
-	return genericapi.NewApisWebService(s.Serializer, APIGroupPrefix, func(req *restful.Request) []metav1.APIGroup {
-		s.apiGroupsForDiscoveryLock.RLock()
-		defer s.apiGroupsForDiscoveryLock.RUnlock()
-
-		sortedGroups := []metav1.APIGroup{}
-
-		// ranging over apiGroupNamesForDiscovery preserves the registration order
-		for _, groupName := range s.apiGroupNamesForDiscovery {
-			sortedGroups = append(sortedGroups, s.apiGroupsForDiscovery[groupName])
-		}
-
-		clientIP := utilnet.GetClientIP(req.Request)
-		serverCIDR := s.discoveryAddresses.ServerAddressByClientCIDRs(clientIP)
-		groups := make([]metav1.APIGroup, len(sortedGroups))
-		for i := range sortedGroups {
-			groups[i] = sortedGroups[i]
-			groups[i].ServerAddressByClientCIDRs = serverCIDR
-		}
-		return groups
-	})
-}
-
 // NewDefaultAPIGroupInfo returns an APIGroupInfo stubbed with "normal" values
 // exposed for easier composition from other packages
 func NewDefaultAPIGroupInfo(group string, registry *registered.APIRegistrationManager, scheme *runtime.Scheme, parameterCodec runtime.ParameterCodec, codecs serializer.CodecFactory) APIGroupInfo {

staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go

@@ -24,7 +24,6 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
-	"reflect"
 	goruntime "runtime"
 	"testing"
 	"time"
@@ -39,13 +38,13 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
-	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/apiserver/pkg/apis/example"
 	examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
 	"k8s.io/apiserver/pkg/server/mux"
@@ -139,7 +138,7 @@ func TestInstallAPIGroups(t *testing.T) {
 	defer etcdserver.Terminate(t)
 
 	config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix")
-	config.DiscoveryAddresses = DefaultDiscoveryAddresses{DefaultAddress: "ExternalAddress"}
+	config.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: "ExternalAddress"}
 
 	s, err := config.SkipComplete().New()
 	if err != nil {
@@ -461,218 +460,6 @@ func decodeResponse(resp *http.Response, obj interface{}) error {
 	return nil
 }
 
-func getGroupList(server *httptest.Server) (*metav1.APIGroupList, error) {
-	resp, err := http.Get(server.URL + "/apis")
-	if err != nil {
-		return nil, err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("unexpected server response, expected %d, actual: %d", http.StatusOK, resp.StatusCode)
-	}
-
-	groupList := metav1.APIGroupList{}
-	err = decodeResponse(resp, &groupList)
-	return &groupList, err
-}
-
-func TestDiscoveryAtAPIS(t *testing.T) {
-	master, etcdserver, _, assert := newMaster(t)
-	defer etcdserver.Terminate(t)
-
-	server := httptest.NewServer(master.Handler)
-	groupList, err := getGroupList(server)
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	assert.Equal(0, len(groupList.Groups))
-
-	// Add a Group.
-	extensionsVersions := []metav1.GroupVersionForDiscovery{
-		{
-			GroupVersion: examplev1.SchemeGroupVersion.String(),
-			Version:      examplev1.SchemeGroupVersion.Version,
-		},
-	}
-	extensionsPreferredVersion := metav1.GroupVersionForDiscovery{
-		GroupVersion: extensionsGroupName + "/preferred",
-		Version:      "preferred",
-	}
-	master.AddAPIGroupForDiscovery(metav1.APIGroup{
-		Name:             extensionsGroupName,
-		Versions:         extensionsVersions,
-		PreferredVersion: extensionsPreferredVersion,
-	})
-
-	groupList, err = getGroupList(server)
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-
-	assert.Equal(1, len(groupList.Groups))
-	groupListGroup := groupList.Groups[0]
-	assert.Equal(extensionsGroupName, groupListGroup.Name)
-	assert.Equal(extensionsVersions, groupListGroup.Versions)
-	assert.Equal(extensionsPreferredVersion, groupListGroup.PreferredVersion)
-	assert.Equal(master.discoveryAddresses.ServerAddressByClientCIDRs(utilnet.GetClientIP(&http.Request{})), groupListGroup.ServerAddressByClientCIDRs)
-
-	// Remove the group.
-	master.RemoveAPIGroupForDiscovery(extensionsGroupName)
-	groupList, err = getGroupList(server)
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-
-	assert.Equal(0, len(groupList.Groups))
-}
-
-func TestDiscoveryOrdering(t *testing.T) {
-	master, etcdserver, _, assert := newMaster(t)
-	defer etcdserver.Terminate(t)
-
-	server := httptest.NewServer(master.Handler)
-	groupList, err := getGroupList(server)
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	assert.Equal(0, len(groupList.Groups))
-
-	// Register three groups
-	master.AddAPIGroupForDiscovery(metav1.APIGroup{Name: "x"})
-	master.AddAPIGroupForDiscovery(metav1.APIGroup{Name: "y"})
-	master.AddAPIGroupForDiscovery(metav1.APIGroup{Name: "z"})
-	// Register three additional groups that come earlier alphabetically
-	master.AddAPIGroupForDiscovery(metav1.APIGroup{Name: "a"})
-	master.AddAPIGroupForDiscovery(metav1.APIGroup{Name: "b"})
-	master.AddAPIGroupForDiscovery(metav1.APIGroup{Name: "c"})
-	// Make sure re-adding doesn't double-register or make a group lose its place
-	master.AddAPIGroupForDiscovery(metav1.APIGroup{Name: "x"})
-
-	groupList, err = getGroupList(server)
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-
-	assert.Equal(6, len(groupList.Groups))
-	assert.Equal("x", groupList.Groups[0].Name)
-	assert.Equal("y", groupList.Groups[1].Name)
-	assert.Equal("z", groupList.Groups[2].Name)
-	assert.Equal("a", groupList.Groups[3].Name)
-	assert.Equal("b", groupList.Groups[4].Name)
-	assert.Equal("c", groupList.Groups[5].Name)
-
-	// Remove a group.
-	master.RemoveAPIGroupForDiscovery("a")
-	groupList, err = getGroupList(server)
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	assert.Equal(5, len(groupList.Groups))
-
-	// Re-adding should move to the end.
-	master.AddAPIGroupForDiscovery(metav1.APIGroup{Name: "a"})
-	groupList, err = getGroupList(server)
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	assert.Equal(6, len(groupList.Groups))
-	assert.Equal("x", groupList.Groups[0].Name)
-	assert.Equal("y", groupList.Groups[1].Name)
-	assert.Equal("z", groupList.Groups[2].Name)
-	assert.Equal("b", groupList.Groups[3].Name)
-	assert.Equal("c", groupList.Groups[4].Name)
-	assert.Equal("a", groupList.Groups[5].Name)
-}
-
-func TestGetServerAddressByClientCIDRs(t *testing.T) {
-	publicAddressCIDRMap := []metav1.ServerAddressByClientCIDR{
-		{
-			ClientCIDR:    "0.0.0.0/0",
-			ServerAddress: "ExternalAddress",
-		},
-	}
-	internalAddressCIDRMap := []metav1.ServerAddressByClientCIDR{
-		publicAddressCIDRMap[0],
-		{
-			ClientCIDR:    "10.0.0.0/24",
-			ServerAddress: "serviceIP",
-		},
-	}
-	internalIP := "10.0.0.1"
-	publicIP := "1.1.1.1"
-	testCases := []struct {
-		Request     http.Request
-		ExpectedMap []metav1.ServerAddressByClientCIDR
-	}{
-		{
-			Request:     http.Request{},
-			ExpectedMap: publicAddressCIDRMap,
-		},
-		{
-			Request: http.Request{
-				Header: map[string][]string{
-					"X-Real-Ip": {internalIP},
-				},
-			},
-			ExpectedMap: internalAddressCIDRMap,
-		},
-		{
-			Request: http.Request{
-				Header: map[string][]string{
-					"X-Real-Ip": {publicIP},
-				},
-			},
-			ExpectedMap: publicAddressCIDRMap,
-		},
-		{
-			Request: http.Request{
-				Header: map[string][]string{
-					"X-Forwarded-For": {internalIP},
-				},
-			},
-			ExpectedMap: internalAddressCIDRMap,
-		},
-		{
-			Request: http.Request{
-				Header: map[string][]string{
-					"X-Forwarded-For": {publicIP},
-				},
-			},
-			ExpectedMap: publicAddressCIDRMap,
-		},
-
-		{
-			Request: http.Request{
-				RemoteAddr: internalIP,
-			},
-			ExpectedMap: internalAddressCIDRMap,
-		},
-		{
-			Request: http.Request{
-				RemoteAddr: publicIP,
-			},
-			ExpectedMap: publicAddressCIDRMap,
-		},
-		{
-			Request: http.Request{
-				RemoteAddr: "invalidIP",
-			},
-			ExpectedMap: publicAddressCIDRMap,
-		},
-	}
-
-	_, ipRange, _ := net.ParseCIDR("10.0.0.0/24")
-	discoveryAddresses := DefaultDiscoveryAddresses{DefaultAddress: "ExternalAddress"}
-	discoveryAddresses.DiscoveryCIDRRules = append(discoveryAddresses.DiscoveryCIDRRules,
-		DiscoveryCIDRRule{IPRange: *ipRange, Address: "serviceIP"})
-
-	for i, test := range testCases {
-		if a, e := discoveryAddresses.ServerAddressByClientCIDRs(utilnet.GetClientIP(&test.Request)), test.ExpectedMap; reflect.DeepEqual(e, a) != true {
-			t.Fatalf("test case %d failed. expected: %v, actual: %v", i+1, e, a)
-		}
-	}
-}
-
 type testGetterStorage struct {
 	Version string
 }