Merge pull request #45182 from deads2k/tpr-08-simple-serving

hack/.linted_packages

@@ -408,6 +408,7 @@ staging/src/k8s.io/kube-apiextensions-server/pkg/client/informers/internalversio
 staging/src/k8s.io/kube-apiextensions-server/pkg/client/informers/internalversion/apiextensions/internalversion
 staging/src/k8s.io/kube-apiextensions-server/pkg/client/listers/apiextensions/internalversion
 staging/src/k8s.io/kube-apiextensions-server/pkg/client/listers/apiextensions/v1alpha1
+staging/src/k8s.io/kube-apiextensions-server/test/integration
 staging/src/k8s.io/metrics/pkg/apis/custom_metrics/install
 staging/src/k8s.io/metrics/pkg/apis/metrics/install
 staging/src/k8s.io/sample-apiserver

hack/make-rules/test-integration.sh

@@ -44,6 +44,9 @@ kube::test::find_integration_test_dirs() {
     find test/integration/ -name '*_test.go' -print0 \
       | xargs -0n1 dirname | sed "s|^|${KUBE_GO_PACKAGE}/|" \
       | LC_ALL=C sort -u
+    find vendor/k8s.io/kube-apiextensions-server/test/integration/ -name '*_test.go' -print0 \
+      | xargs -0n1 dirname | sed "s|^|${KUBE_GO_PACKAGE}/|" \
+      | LC_ALL=C sort -u
   )
 }
 

hack/make-rules/test.sh

@@ -72,7 +72,11 @@ kube::test::find_dirs() {
     find ./staging/src/k8s.io/kube-aggregator -name '*_test.go' \
       -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./staging/src/|./vendor/|' | LC_ALL=C sort -u
 
-    find ./staging/src/k8s.io/kube-apiextensions-server -name '*_test.go' \
+    find ./staging/src/k8s.io/kube-apiextensions-server -not \( \
+        \( \
+          -o -path './test/integration/*' \
+        \) -prune \
+      \) -name '*_test.go' \
       -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./staging/src/|./vendor/|' | LC_ALL=C sort -u
 
     find ./staging/src/k8s.io/sample-apiserver -name '*_test.go' \

staging/src/k8s.io/apiserver/pkg/endpoints/discovery/group.go

@@ -28,15 +28,15 @@ import (
 	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 )
 
-// apiGroupHandler creates a webservice serving the supported versions, preferred version, and name
+// APIGroupHandler creates a webservice serving the supported versions, preferred version, and name
 // of a group. E.g., such a web service will be registered at /apis/extensions.
-type apiGroupHandler struct {
+type APIGroupHandler struct {
 	serializer runtime.NegotiatedSerializer
 
 	group metav1.APIGroup
 }
 
-func NewAPIGroupHandler(serializer runtime.NegotiatedSerializer, group metav1.APIGroup) *apiGroupHandler {
+func NewAPIGroupHandler(serializer runtime.NegotiatedSerializer, group metav1.APIGroup) *APIGroupHandler {
 	if keepUnversioned(group.Name) {
 		// Because in release 1.1, /apis/extensions returns response with empty
 		// APIVersion, we use stripVersionNegotiatedSerializer to keep the
@@ -44,13 +44,13 @@ func NewAPIGroupHandler(serializer runtime.NegotiatedSerializer, group metav1.AP
 		serializer = stripVersionNegotiatedSerializer{serializer}
 	}
 
-	return &apiGroupHandler{
+	return &APIGroupHandler{
 		serializer: serializer,
 		group:      group,
 	}
 }
 
-func (s *apiGroupHandler) WebService() *restful.WebService {
+func (s *APIGroupHandler) WebService() *restful.WebService {
 	mediaTypes, _ := negotiation.MediaTypesForSerializer(s.serializer)
 	ws := new(restful.WebService)
 	ws.Path(APIGroupPrefix + "/" + s.group.Name)
@@ -65,6 +65,10 @@ func (s *apiGroupHandler) WebService() *restful.WebService {
 }
 
 // handle returns a handler which will return the api.GroupAndVersion of the group.
-func (s *apiGroupHandler) handle(req *restful.Request, resp *restful.Response) {
-	responsewriters.WriteObjectNegotiated(s.serializer, schema.GroupVersion{}, resp.ResponseWriter, req.Request, http.StatusOK, &s.group)
+func (s *APIGroupHandler) handle(req *restful.Request, resp *restful.Response) {
+	s.ServeHTTP(resp.ResponseWriter, req.Request)
+}
+
+func (s *APIGroupHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	responsewriters.WriteObjectNegotiated(s.serializer, schema.GroupVersion{}, w, req, http.StatusOK, &s.group)
 }

staging/src/k8s.io/apiserver/pkg/endpoints/discovery/version.go

@@ -32,16 +32,22 @@ type APIResourceLister interface {
 	ListAPIResources() []metav1.APIResource
 }
 
-// apiVersionHandler creates a webservice serving the supported resources for the version
+type APIResourceListerFunc func() []metav1.APIResource
+
+func (f APIResourceListerFunc) ListAPIResources() []metav1.APIResource {
+	return f()
+}
+
+// APIVersionHandler creates a webservice serving the supported resources for the version
 // E.g., such a web service will be registered at /apis/extensions/v1beta1.
-type apiVersionHandler struct {
+type APIVersionHandler struct {
 	serializer runtime.NegotiatedSerializer
 
 	groupVersion      schema.GroupVersion
 	apiResourceLister APIResourceLister
 }
 
-func NewAPIVersionHandler(serializer runtime.NegotiatedSerializer, groupVersion schema.GroupVersion, apiResourceLister APIResourceLister) *apiVersionHandler {
+func NewAPIVersionHandler(serializer runtime.NegotiatedSerializer, groupVersion schema.GroupVersion, apiResourceLister APIResourceLister) *APIVersionHandler {
 	if keepUnversioned(groupVersion.Group) {
 		// Because in release 1.1, /apis/extensions returns response with empty
 		// APIVersion, we use stripVersionNegotiatedSerializer to keep the
@@ -49,14 +55,14 @@ func NewAPIVersionHandler(serializer runtime.NegotiatedSerializer, groupVersion
 		serializer = stripVersionNegotiatedSerializer{serializer}
 	}
 
-	return &apiVersionHandler{
+	return &APIVersionHandler{
 		serializer:        serializer,
 		groupVersion:      groupVersion,
 		apiResourceLister: apiResourceLister,
 	}
 }
 
-func (s *apiVersionHandler) AddToWebService(ws *restful.WebService) {
+func (s *APIVersionHandler) AddToWebService(ws *restful.WebService) {
 	mediaTypes, _ := negotiation.MediaTypesForSerializer(s.serializer)
 	ws.Route(ws.GET("/").To(s.handle).
 		Doc("get available resources").
@@ -67,7 +73,11 @@ func (s *apiVersionHandler) AddToWebService(ws *restful.WebService) {
 }
 
 // handle returns a handler which will return the api.VersionAndVersion of the group.
-func (s *apiVersionHandler) handle(req *restful.Request, resp *restful.Response) {
-	responsewriters.WriteObjectNegotiated(s.serializer, schema.GroupVersion{}, resp.ResponseWriter, req.Request, http.StatusOK,
+func (s *APIVersionHandler) handle(req *restful.Request, resp *restful.Response) {
+	s.ServeHTTP(resp.ResponseWriter, req.Request)
+}
+
+func (s *APIVersionHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	responsewriters.WriteObjectNegotiated(s.serializer, schema.GroupVersion{}, w, req, http.StatusOK,
 		&metav1.APIResourceList{GroupVersion: s.groupVersion.String(), APIResources: s.apiResourceLister.ListAPIResources()})
 }

staging/src/k8s.io/kube-apiextensions-server/artifacts/customresource-01/noxu-apiservice.yaml

+apiVersion: apiregistration.k8s.io/v1alpha1
+kind: APIService
+metadata:
+  name: v1alpha1.mygroup.example.com
+spec:
+  insecureSkipTLSVerify: true
+  group: mygroup.example.com
+  priority: 500
+  service:
+    name: api
+    namespace: kube-apiextensions
+  version: v1alpha1

staging/src/k8s.io/kube-apiextensions-server/artifacts/customresource-01/noxu-resource-definition.yaml

@@ -5,6 +5,7 @@ metadata:
 spec:
   group: mygroup.example.com
   version: v1alpha1
+  scope: Namespaced
   names:
     name: noxus
     singular: noxu

staging/src/k8s.io/kube-apiextensions-server/artifacts/customresource-01/noxu.yaml

+apiVersion: mygroup.example.com/v1alpha1
+kind: Noxu
+metadata:
+  name: alfa-noxu
+spec:
+  key: value
\ No newline at end of file

staging/src/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/types.go

@@ -33,7 +33,7 @@ type CustomResourceSpec struct {
 
 // CustomResourceNames indicates the names to serve this CustomResource
 type CustomResourceNames struct {
-	// Plural is the plural name of the resource to serve.  It must match the name of the TPR-registration
+	// Plural is the plural name of the resource to serve.  It must match the name of the CustomResource-registration
 	// too: plural.group and it must be all lowercase.
 	Plural string
 	// Singular is the singular name of the resource.  It must be all lowercase  Defaults to lowercased <kind>

staging/src/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1/types.go

@@ -33,7 +33,7 @@ type CustomResourceSpec struct {
 
 // CustomResourceNames indicates the names to serve this CustomResource
 type CustomResourceNames struct {
-	// Plural is the plural name of the resource to serve.  It must match the name of the TPR-registration
+	// Plural is the plural name of the resource to serve.  It must match the name of the CustomResource-registration
 	// too: plural.group and it must be all lowercase.
 	Plural string `json:"plural" protobuf:"bytes,1,opt,name=plural"`
 	// Singular is the singular name of the resource.  It must be all lowercase  Defaults to lowercased <kind>

staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/BUILD

@@ -9,18 +9,42 @@ load(
 
 go_library(
     name = "go_default_library",
-    srcs = ["apiserver.go"],
+    srcs = [
+        "apiserver.go",
+        "customresource_discovery.go",
+        "customresource_discovery_controller.go",
+        "customresource_handler.go",
+    ],
     tags = ["automanaged"],
     deps = [
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/handlers:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/registry/rest:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
+        "//vendor/k8s.io/client-go/discovery:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
         "//vendor/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions:go_default_library",
         "//vendor/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/install:go_default_library",
         "//vendor/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1:go_default_library",
@@ -28,6 +52,9 @@ go_library(
         "//vendor/k8s.io/kube-apiextensions-server/pkg/client/clientset/internalclientset:go_default_library",
         "//vendor/k8s.io/kube-apiextensions-server/pkg/client/informers/externalversions:go_default_library",
         "//vendor/k8s.io/kube-apiextensions-server/pkg/client/informers/internalversion:go_default_library",
+        "//vendor/k8s.io/kube-apiextensions-server/pkg/client/informers/internalversion/apiextensions/internalversion:go_default_library",
+        "//vendor/k8s.io/kube-apiextensions-server/pkg/client/listers/apiextensions/internalversion:go_default_library",
         "//vendor/k8s.io/kube-apiextensions-server/pkg/registry/customresource:go_default_library",
+        "//vendor/k8s.io/kube-apiextensions-server/pkg/registry/customresourcestorage:go_default_library",
     ],
 )

staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/apiserver.go

@@ -17,6 +17,9 @@ limitations under the License.
 package apiserver
 
 import (
+	"net/http"
+	"time"
+
 	"k8s.io/apimachinery/pkg/apimachinery/announced"
 	"k8s.io/apimachinery/pkg/apimachinery/registered"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -24,17 +27,20 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
+	genericregistry "k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/registry/rest"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 
 	"k8s.io/kube-apiextensions-server/pkg/apis/apiextensions"
 	"k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/install"
 	"k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1"
+	"k8s.io/kube-apiextensions-server/pkg/client/clientset/internalclientset"
+	internalinformers "k8s.io/kube-apiextensions-server/pkg/client/informers/internalversion"
 	"k8s.io/kube-apiextensions-server/pkg/registry/customresource"
 
 	// make sure the generated client works
 	_ "k8s.io/kube-apiextensions-server/pkg/client/clientset/clientset"
-	_ "k8s.io/kube-apiextensions-server/pkg/client/clientset/internalclientset"
 	_ "k8s.io/kube-apiextensions-server/pkg/client/informers/externalversions"
 	_ "k8s.io/kube-apiextensions-server/pkg/client/informers/internalversion"
 )
@@ -64,6 +70,8 @@ func init() {
 
 type Config struct {
 	GenericConfig *genericapiserver.Config
+
+	CustomResourceRESTOptionsGetter genericregistry.RESTOptionsGetter
 }
 
 type CustomResources struct {
@@ -76,6 +84,7 @@ type completedConfig struct {
 
 // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
 func (c *Config) Complete() completedConfig {
+	c.GenericConfig.EnableDiscovery = false
 	c.GenericConfig.Complete()
 
 	c.GenericConfig.Version = &version.Info{
@@ -92,7 +101,7 @@ func (c *Config) SkipComplete() completedConfig {
 }
 
 // New returns a new instance of CustomResources from the given config.
-func (c completedConfig) New() (*CustomResources, error) {
+func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget, stopCh <-chan struct{}) (*CustomResources, error) {
 	genericServer, err := c.Config.GenericConfig.SkipComplete().New() // completion is done in Complete, no need for a second time
 	if err != nil {
 		return nil, err
@@ -112,5 +121,47 @@ func (c completedConfig) New() (*CustomResources, error) {
 		return nil, err
 	}
 
+	customResourceClient, err := internalclientset.NewForConfig(s.GenericAPIServer.LoopbackClientConfig)
+	if err != nil {
+		return nil, err
+	}
+	customResourceInformers := internalinformers.NewSharedInformerFactory(customResourceClient, 5*time.Minute)
+
+	delegateHandler := delegationTarget.UnprotectedHandler()
+	if delegateHandler == nil {
+		delegateHandler = http.NotFoundHandler()
+	}
+
+	versionDiscoveryHandler := &versionDiscoveryHandler{
+		discovery: map[schema.GroupVersion]*discovery.APIVersionHandler{},
+		delegate:  delegateHandler,
+	}
+	groupDiscoveryHandler := &groupDiscoveryHandler{
+		discovery: map[string]*discovery.APIGroupHandler{},
+		delegate:  delegateHandler,
+	}
+	customResourceHandler := NewCustomResourceHandler(
+		versionDiscoveryHandler,
+		groupDiscoveryHandler,
+		s.GenericAPIServer.RequestContextMapper(),
+		customResourceInformers.Apiextensions().InternalVersion().CustomResources().Lister(),
+		delegationTarget.UnprotectedHandler(),
+		c.CustomResourceRESTOptionsGetter,
+		c.GenericConfig.AdmissionControl,
+	)
+	s.GenericAPIServer.FallThroughHandler.Handle("/apis", customResourceHandler)
+	s.GenericAPIServer.FallThroughHandler.HandlePrefix("/apis/", customResourceHandler)
+
+	customResourceController := NewDiscoveryController(customResourceInformers.Apiextensions().InternalVersion().CustomResources(), versionDiscoveryHandler, groupDiscoveryHandler)
+
+	s.GenericAPIServer.AddPostStartHook("start-apiextensions-informers", func(context genericapiserver.PostStartHookContext) error {
+		customResourceInformers.Start(stopCh)
+		return nil
+	})
+	s.GenericAPIServer.AddPostStartHook("start-apiextensions-controllers", func(context genericapiserver.PostStartHookContext) error {
+		go customResourceController.Run(stopCh)
+		return nil
+	})
+
 	return s, nil
 }

staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_discovery.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"net/http"
+	"strings"
+	"sync"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
+)
+
+type versionDiscoveryHandler struct {
+	// TODO, writing is infrequent, optimize this
+	discoveryLock sync.RWMutex
+	discovery     map[schema.GroupVersion]*discovery.APIVersionHandler
+
+	delegate http.Handler
+}
+
+func (r *versionDiscoveryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	pathParts := splitPath(req.URL.Path)
+	// only match /apis/<group>/<version>
+	if len(pathParts) != 3 || pathParts[0] != "apis" {
+		r.delegate.ServeHTTP(w, req)
+		return
+	}
+	discovery, ok := r.getDiscovery(schema.GroupVersion{Group: pathParts[1], Version: pathParts[2]})
+	if !ok {
+		r.delegate.ServeHTTP(w, req)
+		return
+	}
+
+	discovery.ServeHTTP(w, req)
+}
+
+func (r *versionDiscoveryHandler) getDiscovery(gv schema.GroupVersion) (*discovery.APIVersionHandler, bool) {
+	r.discoveryLock.RLock()
+	defer r.discoveryLock.RUnlock()
+
+	ret, ok := r.discovery[gv]
+	return ret, ok
+}
+
+func (r *versionDiscoveryHandler) setDiscovery(gv schema.GroupVersion, discovery *discovery.APIVersionHandler) {
+	r.discoveryLock.Lock()
+	defer r.discoveryLock.Unlock()
+
+	r.discovery[gv] = discovery
+}
+
+func (r *versionDiscoveryHandler) unsetDiscovery(gv schema.GroupVersion) {
+	r.discoveryLock.Lock()
+	defer r.discoveryLock.Unlock()
+
+	delete(r.discovery, gv)
+}
+
+type groupDiscoveryHandler struct {
+	// TODO, writing is infrequent, optimize this
+	discoveryLock sync.RWMutex
+	discovery     map[string]*discovery.APIGroupHandler
+
+	delegate http.Handler
+}
+
+func (r *groupDiscoveryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	pathParts := splitPath(req.URL.Path)
+	// only match /apis/<group>
+	if len(pathParts) != 2 || pathParts[0] != "apis" {
+		r.delegate.ServeHTTP(w, req)
+		return
+	}
+	discovery, ok := r.getDiscovery(pathParts[1])
+	if !ok {
+		r.delegate.ServeHTTP(w, req)
+		return
+	}
+
+	discovery.ServeHTTP(w, req)
+}
+
+func (r *groupDiscoveryHandler) getDiscovery(group string) (*discovery.APIGroupHandler, bool) {
+	r.discoveryLock.RLock()
+	defer r.discoveryLock.RUnlock()
+
+	ret, ok := r.discovery[group]
+	return ret, ok
+}
+
+func (r *groupDiscoveryHandler) setDiscovery(group string, discovery *discovery.APIGroupHandler) {
+	r.discoveryLock.Lock()
+	defer r.discoveryLock.Unlock()
+
+	r.discovery[group] = discovery
+}
+
+func (r *groupDiscoveryHandler) unsetDiscovery(group string) {
+	r.discoveryLock.Lock()
+	defer r.discoveryLock.Unlock()
+
+	delete(r.discovery, group)
+}
+
+// splitPath returns the segments for a URL path.
+func splitPath(path string) []string {
+	path = strings.Trim(path, "/")
+	if path == "" {
+		return []string{}
+	}
+	return strings.Split(path, "/")
+}

staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_discovery_controller.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/golang/glog"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+
+	"k8s.io/kube-apiextensions-server/pkg/apis/apiextensions"
+	informers "k8s.io/kube-apiextensions-server/pkg/client/informers/internalversion/apiextensions/internalversion"
+	listers "k8s.io/kube-apiextensions-server/pkg/client/listers/apiextensions/internalversion"
+)
+
+type DiscoveryController struct {
+	versionHandler *versionDiscoveryHandler
+	groupHandler   *groupDiscoveryHandler
+
+	customResourceLister  listers.CustomResourceLister
+	customResourcesSynced cache.InformerSynced
+
+	// To allow injection for testing.
+	syncFn func(version schema.GroupVersion) error
+
+	queue workqueue.RateLimitingInterface
+}
+
+func NewDiscoveryController(customResourceInformer informers.CustomResourceInformer, versionHandler *versionDiscoveryHandler, groupHandler *groupDiscoveryHandler) *DiscoveryController {
+	c := &DiscoveryController{
+		versionHandler:        versionHandler,
+		groupHandler:          groupHandler,
+		customResourceLister:  customResourceInformer.Lister(),
+		customResourcesSynced: customResourceInformer.Informer().HasSynced,
+
+		queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DiscoveryController"),
+	}
+
+	customResourceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    c.addCustomResource,
+		UpdateFunc: c.updateCustomResource,
+		DeleteFunc: c.deleteCustomResource,
+	})
+
+	c.syncFn = c.sync
+
+	return c
+}
+
+func (c *DiscoveryController) sync(version schema.GroupVersion) error {
+
+	apiVersionsForDiscovery := []metav1.GroupVersionForDiscovery{}
+	apiResourcesForDiscovery := []metav1.APIResource{}
+
+	customResources, err := c.customResourceLister.List(labels.Everything())
+	if err != nil {
+		return err
+	}
+	foundVersion := false
+	foundGroup := false
+	for _, customResource := range customResources {
+		// TODO add status checking
+
+		if customResource.Spec.Group != version.Group {
+			continue
+		}
+		foundGroup = true
+		apiVersionsForDiscovery = append(apiVersionsForDiscovery, metav1.GroupVersionForDiscovery{
+			GroupVersion: customResource.Spec.Group + "/" + customResource.Spec.Version,
+			Version:      customResource.Spec.Version,
+		})
+
+		if customResource.Spec.Version != version.Version {
+			continue
+		}
+		foundVersion = true
+
+		apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{
+			Name:         customResource.Spec.Names.Plural,
+			SingularName: customResource.Spec.Names.Singular,
+			Namespaced:   customResource.Spec.Scope == apiextensions.NamespaceScoped,
+			Kind:         customResource.Spec.Names.Kind,
+			Verbs:        metav1.Verbs([]string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}),
+			ShortNames:   customResource.Spec.Names.ShortNames,
+		})
+	}
+
+	if !foundGroup {
+		c.groupHandler.unsetDiscovery(version.Group)
+		c.versionHandler.unsetDiscovery(version)
+		return nil
+	}
+
+	apiGroup := metav1.APIGroup{
+		Name:     version.Group,
+		Versions: apiVersionsForDiscovery,
+		// the preferred versions for a group is arbitrary since there cannot be duplicate resources
+		PreferredVersion: apiVersionsForDiscovery[0],
+	}
+	c.groupHandler.setDiscovery(version.Group, discovery.NewAPIGroupHandler(Codecs, apiGroup))
+
+	if !foundVersion {
+		c.versionHandler.unsetDiscovery(version)
+		return nil
+	}
+	c.versionHandler.setDiscovery(version, discovery.NewAPIVersionHandler(Codecs, version, discovery.APIResourceListerFunc(func() []metav1.APIResource {
+		return apiResourcesForDiscovery
+	})))
+
+	return nil
+}
+
+func (c *DiscoveryController) Run(stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+	defer glog.Infof("Shutting down DiscoveryController")
+
+	glog.Infof("Starting DiscoveryController")
+
+	if !cache.WaitForCacheSync(stopCh, c.customResourcesSynced) {
+		utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync"))
+		return
+	}
+
+	// only start one worker thread since its a slow moving API
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	<-stopCh
+}
+
+func (c *DiscoveryController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+// processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
+func (c *DiscoveryController) processNextWorkItem() bool {
+	key, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(key)
+
+	err := c.syncFn(key.(schema.GroupVersion))
+	if err == nil {
+		c.queue.Forget(key)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with: %v", key, err))
+	c.queue.AddRateLimited(key)
+
+	return true
+}
+
+func (c *DiscoveryController) enqueue(obj *apiextensions.CustomResource) {
+	c.queue.Add(schema.GroupVersion{Group: obj.Spec.Group, Version: obj.Spec.Version})
+}
+
+func (c *DiscoveryController) addCustomResource(obj interface{}) {
+	castObj := obj.(*apiextensions.CustomResource)
+	glog.V(4).Infof("Adding customresource %s", castObj.Name)
+	c.enqueue(castObj)
+}
+
+func (c *DiscoveryController) updateCustomResource(obj, _ interface{}) {
+	castObj := obj.(*apiextensions.CustomResource)
+	glog.V(4).Infof("Updating customresource %s", castObj.Name)
+	c.enqueue(castObj)
+}
+
+func (c *DiscoveryController) deleteCustomResource(obj interface{}) {
+	castObj, ok := obj.(*apiextensions.CustomResource)
+	if !ok {
+		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			glog.Errorf("Couldn't get object from tombstone %#v", obj)
+			return
+		}
+		castObj, ok = tombstone.Obj.(*apiextensions.CustomResource)
+		if !ok {
+			glog.Errorf("Tombstone contained object that is not expected %#v", obj)
+			return
+		}
+	}
+	glog.V(4).Infof("Deleting customresource %q", castObj.Name)
+	c.enqueue(castObj)
+}

staging/src/k8s.io/kube-apiextensions-server/pkg/cmd/server/BUILD

@@ -13,6 +13,7 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//vendor/github.com/spf13/cobra:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
         "//vendor/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1:go_default_library",

staging/src/k8s.io/kube-apiextensions-server/pkg/cmd/server/start.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1"
@@ -94,8 +95,19 @@ func (o CustomResourcesServerOptions) Config() (*apiserver.Config, error) {
 		return nil, err
 	}
 
+	customResourceRESTOptionsGetter := apiserver.CustomResourceRESTOptionsGetter{
+		StorageConfig:           o.RecommendedOptions.Etcd.StorageConfig,
+		StoragePrefix:           o.RecommendedOptions.Etcd.StorageConfig.Prefix,
+		EnableWatchCache:        o.RecommendedOptions.Etcd.EnableWatchCache,
+		EnableGarbageCollection: o.RecommendedOptions.Etcd.EnableGarbageCollection,
+		DeleteCollectionWorkers: o.RecommendedOptions.Etcd.DeleteCollectionWorkers,
+	}
+	customResourceRESTOptionsGetter.StorageConfig.Codec = unstructured.UnstructuredJSONScheme
+	customResourceRESTOptionsGetter.StorageConfig.Copier = apiserver.UnstructuredCopier{}
+
 	config := &apiserver.Config{
-		GenericConfig: serverConfig,
+		GenericConfig:                   serverConfig,
+		CustomResourceRESTOptionsGetter: customResourceRESTOptionsGetter,
 	}
 	return config, nil
 }
@@ -106,7 +118,7 @@ func (o CustomResourcesServerOptions) RunCustomResourcesServer(stopCh <-chan str
 		return err
 	}
 
-	server, err := config.Complete().New()
+	server, err := config.Complete().New(genericapiserver.EmptyDelegate, stopCh)
 	if err != nil {
 		return err
 	}

staging/src/k8s.io/kube-apiextensions-server/pkg/registry/customresourcestorage/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "etcd.go",
+        "strategy.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/validation:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/names:go_default_library",
+    ],
+)

staging/src/k8s.io/kube-apiextensions-server/pkg/registry/customresourcestorage/etcd.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package customresourcestorage
+
+import (
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/registry/generic"
+	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
+)
+
+// rest implements a RESTStorage for API services against etcd
+type REST struct {
+	*genericregistry.Store
+}
+
+// NewREST returns a RESTStorage object that will work against API services.
+func NewREST(resource schema.GroupResource, listKind schema.GroupVersionKind, copier runtime.ObjectCopier, strategy CustomResourceStorageStrategy, optsGetter generic.RESTOptionsGetter) *REST {
+	store := &genericregistry.Store{
+		Copier:  copier,
+		NewFunc: func() runtime.Object { return &unstructured.Unstructured{} },
+		NewListFunc: func() runtime.Object {
+			// lists are never stored, only manufactured, so stomp in the right kind
+			ret := &unstructured.UnstructuredList{}
+			ret.SetGroupVersionKind(listKind)
+			return ret
+		},
+		ObjectNameFunc: func(obj runtime.Object) (string, error) {
+			accessor, err := meta.Accessor(obj)
+			if err != nil {
+				return "", err
+			}
+			return accessor.GetName(), nil
+		},
+		PredicateFunc:     strategy.MatchCustomResourceStorage,
+		QualifiedResource: resource,
+
+		CreateStrategy: strategy,
+		UpdateStrategy: strategy,
+		DeleteStrategy: strategy,
+	}
+	options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: strategy.GetAttrs}
+	if err := store.CompleteWithOptions(options); err != nil {
+		panic(err) // TODO: Propagate error up
+	}
+	return &REST{store}
+}

staging/src/k8s.io/kube-apiextensions-server/pkg/registry/customresourcestorage/strategy.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package customresourcestorage
+
+import (
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/api/validation"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/names"
+)
+
+type CustomResourceStorageStrategy struct {
+	runtime.ObjectTyper
+	names.NameGenerator
+
+	namespaceScoped bool
+}
+
+func NewStrategy(typer runtime.ObjectTyper, namespaceScoped bool) CustomResourceStorageStrategy {
+	return CustomResourceStorageStrategy{typer, names.SimpleNameGenerator, namespaceScoped}
+}
+
+func (a CustomResourceStorageStrategy) NamespaceScoped() bool {
+	return a.namespaceScoped
+}
+
+func (CustomResourceStorageStrategy) PrepareForCreate(ctx genericapirequest.Context, obj runtime.Object) {
+}
+
+func (CustomResourceStorageStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj, old runtime.Object) {
+}
+
+func (a CustomResourceStorageStrategy) Validate(ctx genericapirequest.Context, obj runtime.Object) field.ErrorList {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return field.ErrorList{field.Invalid(field.NewPath("metadata"), nil, err.Error())}
+	}
+
+	return validation.ValidateObjectMetaAccessor(accessor, a.namespaceScoped, validation.NameIsDNSSubdomain, field.NewPath("metadata"))
+}
+
+func (CustomResourceStorageStrategy) AllowCreateOnUpdate() bool {
+	return false
+}
+
+func (CustomResourceStorageStrategy) AllowUnconditionalUpdate() bool {
+	return false
+}
+
+func (CustomResourceStorageStrategy) Canonicalize(obj runtime.Object) {
+}
+
+func (CustomResourceStorageStrategy) ValidateUpdate(ctx genericapirequest.Context, obj, old runtime.Object) field.ErrorList {
+	objAccessor, err := meta.Accessor(obj)
+	if err != nil {
+		return field.ErrorList{field.Invalid(field.NewPath("metadata"), nil, err.Error())}
+	}
+	oldAccessor, err := meta.Accessor(old)
+	if err != nil {
+		return field.ErrorList{field.Invalid(field.NewPath("metadata"), nil, err.Error())}
+	}
+
+	return validation.ValidateObjectMetaAccessorUpdate(objAccessor, oldAccessor, field.NewPath("metadata"))
+
+	return field.ErrorList{}
+}
+
+func (a CustomResourceStorageStrategy) GetAttrs(obj runtime.Object) (labels.Set, fields.Set, error) {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return nil, nil, err
+	}
+	return labels.Set(accessor.GetLabels()), objectMetaFieldsSet(accessor, a.namespaceScoped), nil
+}
+
+// objectMetaFieldsSet returns a fields that represent the ObjectMeta.
+func objectMetaFieldsSet(objectMeta metav1.Object, namespaceScoped bool) fields.Set {
+	if namespaceScoped {
+		return fields.Set{
+			"metadata.name":      objectMeta.GetName(),
+			"metadata.namespace": objectMeta.GetNamespace(),
+		}
+	}
+	return fields.Set{
+		"metadata.name": objectMeta.GetName(),
+	}
+}
+
+func (a CustomResourceStorageStrategy) MatchCustomResourceStorage(label labels.Selector, field fields.Selector) storage.SelectionPredicate {
+	return storage.SelectionPredicate{
+		Label:    label,
+		Field:    field,
+		GetAttrs: a.GetAttrs,
+	}
+}

staging/src/k8s.io/kube-apiextensions-server/test/integration/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["basic_test.go"],
+    tags = [
+        "automanaged",
+        "integration",
+    ],
+    deps = [
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1:go_default_library",
+        "//vendor/k8s.io/kube-apiextensions-server/test/integration/testserver:go_default_library",
+    ],
+)

staging/src/k8s.io/kube-apiextensions-server/test/integration/apiserver.local.config/certificates/apiserver.crt

+-----BEGIN CERTIFICATE-----
+MIIDEzCCAfugAwIBAgIBATANBgkqhkiG9w0BAQsFADAfMR0wGwYDVQQDDBRsb2Nh
+bGhvc3RAMTQ5MzY2NDQ4OTAeFw0xNzA1MDExODQ4MDlaFw0xODA1MDExODQ4MDla
+MB8xHTAbBgNVBAMMFGxvY2FsaG9zdEAxNDkzNjY0NDg5MIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEAy6tMZcDbG1J4vbX+YdiPswxqO+hX1r+i9+DFb1N/
+xyodBbprn1Mmd2k1lv3AkiZKm38v7dgzQ9/teA8Jm/1tyjjZSV/CxsZZWNuukPGU
+ykEtn4mvkb5tOI1159ieTBiL4mKx5VNq8DkIpy9CT22Ud9dHkJaxJHcIF601hXHg
+GIRla/6CRlkY/GFUItl1oij4sgzXRTS2pdv8lsmt2s7dXj737l10QCz9YDVuGSfu
+rYoHGwY5ofYYFWzscD7Ds4O0tPdu4mSPIu753K7nB3ilfBi+tUWcSXpw9wE4+hIF
+a1In8jnM+lw5/j/UoghrCtQ54BGWzpivPPXKv2dlNIOPiwIDAQABo1owWDAOBgNV
+HQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB
+/zAgBgNVHREEGTAXgglsb2NhbGhvc3SHBH8AAAGHBH8AAAEwDQYJKoZIhvcNAQEL
+BQADggEBAJbxTi/0Joxx/oja4QDksbWroip0qVJKh1ic7ryai52aSBTcHMF9pWiL
+047lL3sL0sN0YavXPUiow4PMTQm14W01ciwuZj5DCCaXnmnGtBy0fy8ifUdQoD/J
+9pvLQMWAsx+GP2XzY+KxYFQairKS7BehEF/d24TgNHPskgc2p2XgK3Z7Ipp7hQrj
+yZiTNromeULT12d5Zuwf+IeDp3aopGyhxCTOoc+RCz4MKLfKov40xjlaA4jVWazd
+ccHWnagwM5lDlXnmCqZRVvyOWaUulJCEzRFfRTHFxKgj6DSPNt00wHXNmQUvjhN/
+YXFAkfKQQEs3qQRXoHAXKquplnLgjyA=
+-----END CERTIFICATE-----

staging/src/k8s.io/kube-apiextensions-server/test/integration/apiserver.local.config/certificates/apiserver.key

+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAy6tMZcDbG1J4vbX+YdiPswxqO+hX1r+i9+DFb1N/xyodBbpr
+n1Mmd2k1lv3AkiZKm38v7dgzQ9/teA8Jm/1tyjjZSV/CxsZZWNuukPGUykEtn4mv
+kb5tOI1159ieTBiL4mKx5VNq8DkIpy9CT22Ud9dHkJaxJHcIF601hXHgGIRla/6C
+RlkY/GFUItl1oij4sgzXRTS2pdv8lsmt2s7dXj737l10QCz9YDVuGSfurYoHGwY5
+ofYYFWzscD7Ds4O0tPdu4mSPIu753K7nB3ilfBi+tUWcSXpw9wE4+hIFa1In8jnM
++lw5/j/UoghrCtQ54BGWzpivPPXKv2dlNIOPiwIDAQABAoIBABy4qWtoCP4PYUuP
+kLIHsiwTwh90on58Q+Uk43LRmaFihPk70tWDCleolJAYdMGneLn4869c383glEJs
+DHTdBlCQN8QrJvKVIiBvymxSRSNIkcB/0CyDaC+jc08gsyIUDBX+yQuH+fqqcFfz
+SCyfTWKhD0yKk6yKxK9iE7wf1PRf5uLtJD6x1vV0NBmsHH++feODjNVsHDRvnwy6
+3KXkgSvfCTQ7qnQPZ/MSsRxWRdMBCnhaQq9qRnJ8bv8XotrCsEG5laMybriyJYXX
+wvr9Dt04ciUD/g3qwIPy1ygMAKE9ya8hivSRURptZxz9SKCenWWihsfIzk4uyOi+
+sVDkJVECgYEA2981ZhbkruG5JhOsWVXTxDlXOrjI0pUqfej0WEp8lwdBMOrTXFs1
+cB8kdSocVC5GnMTbg6bqJrfglbNDOA7sUA8E8APLFFUAAvPwVfnrdDWk+0jK7atu
+2sixQGeIB97Y6ojWfSbjyA/0p3Z0zCTfP5SR6xt4hVWhInB11m5IpO0CgYEA7SKH
+bfWPZ0xfkFdM2X2cWQjt4zYHIUQqAKLPs2OjPTBC8PioNiZlvZq9bNseYR+eyIg6
+P9Kwe4KV/hzOSScf7JYpsN+YQ1+4Y+E63BkhAHXyz7y/vD+DA/Z83oKbelQtJqwq
+W+Zo1OGJrfZwEKK5JWN9HF+KI9Z1iMyZoyw8L1cCgYEAtaLDfj7TVBVs2qPN8U8R
+zjyAbyZP4IcRv0o+8OE345w+oqabTOScVK+lcpUDKhfAhamqniu5q5qjkYextBG/
+7rM5pP29OmKty8KxfJUlia73SA9udMD2pw68PzRIEBhsofPBHUqPSarEtcMJ4ctk
+EiYuFUdwXNXMc6Lr9eTNZlECgYEAsfFJIvAzjdY3l76KwmGJox4aNHdkXkgiJJwH
+s5s+8Tl34g8VWpzxl5e4MSkz4LmzktL2stHM8MGLAEZpXWdog0YjPsBqJ5R6byih
+3GtW4lufutbuIbqe+6hJB0eGmAL2ZqCmoJODcstTXyEf8rvIpw/C4DmpFT9mryKo
+31LgTr0CgYEAuxusmnR2vzZP/RjpjzmZcvIHf4xORG+SXlg3BXsSEd6+g3Rqiy5t
+Q0UkHHwYnYurBmJ2HL1LG9mZwU89D00F/4mJpJuWfwqtqvodIRZ7bimyGGbvKZ1t
+BGLmUssF5MYn75v7E5opxcc51aieW8nUQbop/PPMvWsYLrL/mcJNBpA=
+-----END RSA PRIVATE KEY-----

staging/src/k8s.io/kube-apiextensions-server/test/integration/basic_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/watch"
+	apiextensionsv1alpha1 "k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1"
+	"k8s.io/kube-apiextensions-server/test/integration/testserver"
+)
+
+func TestServerUp(t *testing.T) {
+	stopCh, _, _, err := testserver.StartDefaultServer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer close(stopCh)
+}
+
+func TestSimpleCRUD(t *testing.T) {
+	stopCh, apiExtensionClient, clientPool, err := testserver.StartDefaultServer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer close(stopCh)
+
+	noxuDefinition := testserver.NewNoxuCustomResourceDefinition()
+	noxuVersionClient, err := testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, clientPool)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ns := "not-the-default"
+	noxuNamespacedResourceClient := noxuVersionClient.Resource(&metav1.APIResource{
+		Name:       noxuDefinition.Spec.Names.Plural,
+		Namespaced: noxuDefinition.Spec.Scope == apiextensionsv1alpha1.NamespaceScoped,
+	}, ns)
+	initialList, err := noxuNamespacedResourceClient.List(metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := 0, len(initialList.(*unstructured.UnstructuredList).Items); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	initialListTypeMeta, err := meta.TypeAccessor(initialList)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := noxuDefinition.Spec.Group+"/"+noxuDefinition.Spec.Version, initialListTypeMeta.GetAPIVersion(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := noxuDefinition.Spec.Names.ListKind, initialListTypeMeta.GetKind(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	initialListListMeta, err := meta.ListAccessor(initialList)
+	if err != nil {
+		t.Fatal(err)
+	}
+	noxuNamespacedWatch, err := noxuNamespacedResourceClient.Watch(metav1.ListOptions{ResourceVersion: initialListListMeta.GetResourceVersion()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer noxuNamespacedWatch.Stop()
+
+	noxuInstanceToCreate := testserver.NewNoxuInstance(ns, "foo")
+	createdNoxuInstance, err := noxuNamespacedResourceClient.Create(noxuInstanceToCreate)
+	if err != nil {
+		t.Logf("%#v", createdNoxuInstance)
+		t.Fatal(err)
+	}
+	createdObjectMeta, err := meta.Accessor(createdNoxuInstance)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// it should have a UUID
+	if len(createdObjectMeta.GetUID()) == 0 {
+		t.Errorf("missing uuid: %#v", createdNoxuInstance)
+	}
+	createdTypeMeta, err := meta.TypeAccessor(createdNoxuInstance)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := noxuDefinition.Spec.Group+"/"+noxuDefinition.Spec.Version, createdTypeMeta.GetAPIVersion(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := noxuDefinition.Spec.Names.Kind, createdTypeMeta.GetKind(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	select {
+	case watchEvent := <-noxuNamespacedWatch.ResultChan():
+		if e, a := watch.Added, watchEvent.Type; e != a {
+			t.Errorf("expected %v, got %v", e, a)
+			break
+		}
+		createdObjectMeta, err := meta.Accessor(watchEvent.Object)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// it should have a UUID
+		if len(createdObjectMeta.GetUID()) == 0 {
+			t.Errorf("missing uuid: %#v", watchEvent.Object)
+		}
+		createdTypeMeta, err := meta.TypeAccessor(watchEvent.Object)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if e, a := noxuDefinition.Spec.Group+"/"+noxuDefinition.Spec.Version, createdTypeMeta.GetAPIVersion(); e != a {
+			t.Errorf("expected %v, got %v", e, a)
+		}
+		if e, a := noxuDefinition.Spec.Names.Kind, createdTypeMeta.GetKind(); e != a {
+			t.Errorf("expected %v, got %v", e, a)
+		}
+
+	case <-time.After(5 * time.Second):
+		t.Errorf("missing watch event")
+	}
+
+	gottenNoxuInstance, err := noxuNamespacedResourceClient.Get("foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := createdNoxuInstance, gottenNoxuInstance; !reflect.DeepEqual(e, a) {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	listWithItem, err := noxuNamespacedResourceClient.List(metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := 1, len(listWithItem.(*unstructured.UnstructuredList).Items); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := *createdNoxuInstance, listWithItem.(*unstructured.UnstructuredList).Items[0]; !reflect.DeepEqual(e, a) {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	if err := noxuNamespacedResourceClient.Delete("foo", nil); err != nil {
+		t.Fatal(err)
+	}
+
+	listWithoutItem, err := noxuNamespacedResourceClient.List(metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e, a := 0, len(listWithoutItem.(*unstructured.UnstructuredList).Items); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	select {
+	case watchEvent := <-noxuNamespacedWatch.ResultChan():
+		if e, a := watch.Deleted, watchEvent.Type; e != a {
+			t.Errorf("expected %v, got %v", e, a)
+			break
+		}
+		deletedObjectMeta, err := meta.Accessor(watchEvent.Object)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// it should have a UUID
+		if e, a := createdObjectMeta.GetUID(), deletedObjectMeta.GetUID(); e != a {
+			t.Errorf("expected %v, got %v", e, a)
+		}
+
+	case <-time.After(5 * time.Second):
+		t.Errorf("missing watch event")
+	}
+}

staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "resources.go",
+        "start.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//vendor/github.com/pborman/uuid:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
+        "//vendor/k8s.io/client-go/dynamic:go_default_library",
+        "//vendor/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1:go_default_library",
+        "//vendor/k8s.io/kube-apiextensions-server/pkg/apiserver:go_default_library",
+        "//vendor/k8s.io/kube-apiextensions-server/pkg/client/clientset/clientset:go_default_library",
+        "//vendor/k8s.io/kube-apiextensions-server/pkg/cmd/server:go_default_library",
+    ],
+)

staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/resources.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testserver
+
+import (
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/dynamic"
+	apiextensionsv1alpha1 "k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1"
+	"k8s.io/kube-apiextensions-server/pkg/client/clientset/clientset"
+)
+
+func NewNoxuCustomResourceDefinition() *apiextensionsv1alpha1.CustomResource {
+	return &apiextensionsv1alpha1.CustomResource{
+		ObjectMeta: metav1.ObjectMeta{Name: "noxus.mygroup.example.com"},
+		Spec: apiextensionsv1alpha1.CustomResourceSpec{
+			Group:   "mygroup.example.com",
+			Version: "v1alpha1",
+			Names: apiextensionsv1alpha1.CustomResourceNames{
+				Plural:   "noxus",
+				Singular: "nonenglishnoxu",
+				Kind:     "WishIHadChosenNoxu",
+				ListKind: "NoxuItemList",
+			},
+		},
+	}
+}
+
+func NewNoxuInstance(namespace, name string) *unstructured.Unstructured {
+	return &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "mygroup.example.com/v1alpha1",
+			"kind":       "WishIHadChosenNoxu",
+			"metadata": map[string]interface{}{
+				"namespace": namespace,
+				"name":      name,
+			},
+			"content": map[string]interface{}{
+				"key": "value",
+			},
+		},
+	}
+}
+
+func CreateNewCustomResourceDefinition(customResource *apiextensionsv1alpha1.CustomResource, apiExtensionsClient clientset.Interface, clientPool dynamic.ClientPool) (*dynamic.Client, error) {
+	_, err := apiExtensionsClient.Apiextensions().CustomResources().Create(customResource)
+	if err != nil {
+		return nil, err
+	}
+
+	// wait until the resource appears in discovery
+	err = wait.PollImmediate(30*time.Millisecond, 30*time.Second, func() (bool, error) {
+		resourceList, err := apiExtensionsClient.Discovery().ServerResourcesForGroupVersion(customResource.Spec.Group + "/" + customResource.Spec.Version)
+		if err != nil {
+			return false, nil
+		}
+		for _, resource := range resourceList.APIResources {
+			if resource.Name == customResource.Spec.Names.Plural {
+				return true, nil
+			}
+		}
+		return false, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	dynamicClient, err := clientPool.ClientForGroupVersionResource(schema.GroupVersionResource{Group: customResource.Spec.Group, Version: customResource.Spec.Version, Resource: customResource.Spec.Names.Plural})
+	if err != nil {
+		return nil, err
+	}
+	return dynamicClient, nil
+}

staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/start.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testserver
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/pborman/uuid"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/client-go/dynamic"
+	extensionsapiserver "k8s.io/kube-apiextensions-server/pkg/apiserver"
+	"k8s.io/kube-apiextensions-server/pkg/client/clientset/clientset"
+	"k8s.io/kube-apiextensions-server/pkg/cmd/server"
+)
+
+func DefaultServerConfig() (*extensionsapiserver.Config, error) {
+	port, err := FindFreeLocalPort()
+	if err != nil {
+		return nil, err
+	}
+
+	options := server.NewCustomResourcesServerOptions(os.Stdout, os.Stderr)
+	options.RecommendedOptions.Audit.Path = "-"
+	options.RecommendedOptions.SecureServing.BindPort = port
+	options.RecommendedOptions.Authentication.SkipInClusterLookup = true
+	options.RecommendedOptions.SecureServing.BindAddress = net.ParseIP("127.0.0.1")
+	etcdURL, ok := os.LookupEnv("KUBE_INTEGRATION_ETCD_URL")
+	if !ok {
+		etcdURL = "http://127.0.0.1:2379"
+	}
+	options.RecommendedOptions.Etcd.StorageConfig.ServerList = []string{etcdURL}
+	options.RecommendedOptions.Etcd.StorageConfig.Prefix = uuid.New()
+
+	// TODO stop copying this
+	// because there isn't currently a way to disable authentication or authorization from options
+	// explode options.Config here
+	genericConfig := genericapiserver.NewConfig(extensionsapiserver.Codecs)
+	genericConfig.Authenticator = nil
+	genericConfig.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
+
+	if err := options.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
+		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
+	}
+	if err := options.RecommendedOptions.Etcd.ApplyTo(genericConfig); err != nil {
+		return nil, err
+	}
+	if err := options.RecommendedOptions.SecureServing.ApplyTo(genericConfig); err != nil {
+		return nil, err
+	}
+	if err := options.RecommendedOptions.Audit.ApplyTo(genericConfig); err != nil {
+		return nil, err
+	}
+	if err := options.RecommendedOptions.Features.ApplyTo(genericConfig); err != nil {
+		return nil, err
+	}
+
+	customResourceRESTOptionsGetter := extensionsapiserver.CustomResourceRESTOptionsGetter{
+		StorageConfig:           options.RecommendedOptions.Etcd.StorageConfig,
+		StoragePrefix:           options.RecommendedOptions.Etcd.StorageConfig.Prefix,
+		EnableWatchCache:        options.RecommendedOptions.Etcd.EnableWatchCache,
+		EnableGarbageCollection: options.RecommendedOptions.Etcd.EnableGarbageCollection,
+		DeleteCollectionWorkers: options.RecommendedOptions.Etcd.DeleteCollectionWorkers,
+	}
+	customResourceRESTOptionsGetter.StorageConfig.Codec = unstructured.UnstructuredJSONScheme
+	customResourceRESTOptionsGetter.StorageConfig.Copier = extensionsapiserver.UnstructuredCopier{}
+
+	config := &extensionsapiserver.Config{
+		GenericConfig:                   genericConfig,
+		CustomResourceRESTOptionsGetter: customResourceRESTOptionsGetter,
+	}
+
+	return config, nil
+}
+
+func StartServer(config *extensionsapiserver.Config) (chan struct{}, clientset.Interface, dynamic.ClientPool, error) {
+	stopCh := make(chan struct{})
+	server, err := config.Complete().New(genericapiserver.EmptyDelegate, stopCh)
+	if err != nil {
+		close(stopCh)
+		return nil, nil, nil, err
+	}
+	go func() {
+		err := server.GenericAPIServer.PrepareRun().Run(stopCh)
+		if err != nil {
+			close(stopCh)
+			panic(err)
+		}
+	}()
+
+	// wait until the server is healthy
+	err = wait.PollImmediate(30*time.Millisecond, 30*time.Second, func() (bool, error) {
+		healthClient, err := clientset.NewForConfig(server.GenericAPIServer.LoopbackClientConfig)
+		if err != nil {
+			return false, nil
+		}
+		healthResult := healthClient.Discovery().RESTClient().Get().AbsPath("/healthz").Do()
+		if healthResult.Error() != nil {
+			return false, nil
+		}
+		rawHealth, err := healthResult.Raw()
+		if err != nil {
+			return false, nil
+		}
+		if string(rawHealth) != "ok" {
+			return false, nil
+		}
+
+		return true, nil
+	})
+	if err != nil {
+		close(stopCh)
+		return nil, nil, nil, err
+	}
+
+	apiExtensionsClient, err := clientset.NewForConfig(server.GenericAPIServer.LoopbackClientConfig)
+	if err != nil {
+		close(stopCh)
+		return nil, nil, nil, err
+	}
+
+	bytes, _ := apiExtensionsClient.Discovery().RESTClient().Get().AbsPath("/apis/apiextensions.k8s.io/v1alpha1").DoRaw()
+	fmt.Print(string(bytes))
+
+	return stopCh, apiExtensionsClient, dynamic.NewDynamicClientPool(server.GenericAPIServer.LoopbackClientConfig), nil
+}
+
+func StartDefaultServer() (chan struct{}, clientset.Interface, dynamic.ClientPool, error) {
+	config, err := DefaultServerConfig()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return StartServer(config)
+}
+
+// FindFreeLocalPort returns the number of an available port number on
+// the loopback interface.  Useful for determining the port to launch
+// a server on.  Error handling required - there is a non-zero chance
+// that the returned port number will be bound by another process
+// after this function returns.
+func FindFreeLocalPort() (int, error) {
+	l, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return 0, err
+	}
+	defer l.Close()
+	_, portStr, err := net.SplitHostPort(l.Addr().String())
+	if err != nil {
+		return 0, err
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		return 0, err
+	}
+	return port, nil
+}