Merge pull request #67205 from roycaihw/crd-openapi-spec

hack/import-restrictions.yaml

@@ -123,6 +123,7 @@
   - k8s.io/apiserver
   - k8s.io/client-go
   - k8s.io/klog
+  - k8s.io/kube-openapi
 
 - baseImportPath: "./vendor/k8s.io/kube-openapi/"
   allowedImports:

staging/src/k8s.io/apiextensions-apiserver/BUILD

@@ -50,6 +50,7 @@ filegroup(
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status:all-srcs",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/crdserverscheme:all-srcs",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:all-srcs",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/openapi:all-srcs",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource:all-srcs",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition:all-srcs",
         "//staging/src/k8s.io/apiextensions-apiserver/test/integration:all-srcs",

staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json

@@ -2263,6 +2263,10 @@
 			"Rev": "8139d8cb77af419532b33dfa7dd09fbc5f1d344f"
 		},
 		{
+			"ImportPath": "k8s.io/kube-openapi/pkg/aggregator",
+			"Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d"
+		},
+		{
 			"ImportPath": "k8s.io/kube-openapi/pkg/builder",
 			"Rev": "c59034cc13d587f5ef4e85ca0ade0c1866ae8e1d"
 		},

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD

@@ -36,6 +36,7 @@ go_library(
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/crdserverscheme:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/openapi:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition:go_default_library",

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go

@@ -41,6 +41,7 @@ import (
 	"k8s.io/apiextensions-apiserver/pkg/controller/establish"
 	"k8s.io/apiextensions-apiserver/pkg/controller/finalizer"
 	"k8s.io/apiextensions-apiserver/pkg/controller/status"
+	openapiaggregator "k8s.io/apiextensions-apiserver/pkg/openapi"
 	"k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition"
 
 	_ "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
@@ -204,7 +205,13 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 		return nil
 	})
 	s.GenericAPIServer.AddPostStartHook("start-apiextensions-controllers", func(context genericapiserver.PostStartHookContext) error {
-		go crdController.Run(context.StopCh)
+		// create OpenAPI aggregation manager in the last step because only now genericapiserver's the OpenAPI services and spec is available (after PrepareRun).
+		crdOpenAPIAggregationManager, err := openapiaggregator.NewAggregationManager(s.GenericAPIServer.OpenAPIService, s.GenericAPIServer.OpenAPIVersionedService, s.GenericAPIServer.StaticOpenAPISpec)
+		if err != nil {
+			return err
+		}
+
+		go crdController.Run(context.StopCh, crdOpenAPIAggregationManager)
 		go namingController.Run(context.StopCh)
 		go establishingController.Run(context.StopCh)
 		go finalizingController.Run(5, context.StopCh)

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go

@@ -31,12 +31,16 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/apiserver/pkg/endpoints/discovery"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
 
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	informers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion"
 	listers "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion"
+	apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features"
+	apiextensionsopenapi "k8s.io/apiextensions-apiserver/pkg/openapi"
 )
 
 type DiscoveryController struct {
@@ -50,6 +54,8 @@ type DiscoveryController struct {
 	syncFn func(version schema.GroupVersion) error
 
 	queue workqueue.RateLimitingInterface
+
+	openAPIAggregationManager apiextensionsopenapi.AggregationManager
 }
 
 func NewDiscoveryController(crdInformer informers.CustomResourceDefinitionInformer, versionHandler *versionDiscoveryHandler, groupHandler *groupDiscoveryHandler) *DiscoveryController {
@@ -83,6 +89,7 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 	if err != nil {
 		return err
 	}
+	apiServiceName := version.Group + "." + version.Version
 	foundVersion := false
 	foundGroup := false
 	for _, crd := range crds {
@@ -119,6 +126,33 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 			continue
 		}
 		foundVersion = true
+		if c.openAPIAggregationManager != nil && utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceValidation) {
+			validationSchema, err := getSchemaForVersion(crd, version.Version)
+			if err != nil {
+				return err
+			}
+			// Convert internal CustomResourceValidation to versioned CustomResourceValidation
+			versionedSchema := new(v1beta1.CustomResourceValidation)
+			if validationSchema == nil {
+				versionedSchema = nil
+			} else {
+				if err := v1beta1.Convert_apiextensions_CustomResourceValidation_To_v1beta1_CustomResourceValidation(validationSchema, versionedSchema, nil); err != nil {
+					return err
+				}
+			}
+			// We aggregate the schema even if it's nil as it maybe a removal of the schema for this CRD,
+			// and the aggreated OpenAPI spec should reflect this change.
+			crdspec, etag, err := apiextensionsopenapi.CustomResourceDefinitionOpenAPISpec(&crd.Spec, version.Version, versionedSchema)
+			if err != nil {
+				return err
+			}
+
+			// Add/update the local API service's spec for the CRD in apiExtensionsServer's
+			// openAPIAggregationManager
+			if err := c.openAPIAggregationManager.AddUpdateLocalAPIServiceSpec(apiServiceName, crdspec, etag); err != nil {
+				return err
+			}
+		}
 
 		verbs := metav1.Verbs([]string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"})
 		// if we're terminating we don't allow some verbs
@@ -164,6 +198,14 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 	if !foundGroup {
 		c.groupHandler.unsetDiscovery(version.Group)
 		c.versionHandler.unsetDiscovery(version)
+		if c.openAPIAggregationManager != nil && utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceValidation) {
+			// Remove the local API service for the CRD in apiExtensionsServer's
+			// openAPIAggregationManager.
+			// Note that we don't check if apiServiceName exists in openAPIAggregationManager
+			// because RemoveAPIServiceSpec properly handles non-existing API service by
+			// returning no error.
+			return c.openAPIAggregationManager.RemoveAPIServiceSpec(apiServiceName)
+		}
 		return nil
 	}
 
@@ -180,6 +222,14 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 
 	if !foundVersion {
 		c.versionHandler.unsetDiscovery(version)
+		if c.openAPIAggregationManager != nil && utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceValidation) {
+			// Remove the local API service for the CRD in apiExtensionsServer's
+			// openAPIAggregationManager.
+			// Note that we don't check if apiServiceName exists in openAPIAggregationManager
+			// because RemoveAPIServiceSpec properly handles non-existing API service by
+			// returning no error.
+			return c.openAPIAggregationManager.RemoveAPIServiceSpec(apiServiceName)
+		}
 		return nil
 	}
 	c.versionHandler.setDiscovery(version, discovery.NewAPIVersionHandler(Codecs, version, discovery.APIResourceListerFunc(func() []metav1.APIResource {
@@ -195,13 +245,15 @@ func sortGroupDiscoveryByKubeAwareVersion(gd []metav1.GroupVersionForDiscovery)
 	})
 }
 
-func (c *DiscoveryController) Run(stopCh <-chan struct{}) {
+func (c *DiscoveryController) Run(stopCh <-chan struct{}, crdOpenAPIAggregationManager apiextensionsopenapi.AggregationManager) {
 	defer utilruntime.HandleCrash()
 	defer c.queue.ShutDown()
 	defer klog.Infof("Shutting down DiscoveryController")
 
 	klog.Infof("Starting DiscoveryController")
 
+	c.openAPIAggregationManager = crdOpenAPIAggregationManager
+
 	if !cache.WaitForCacheSync(stopCh, c.crdsSynced) {
 		utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync"))
 		return

staging/src/k8s.io/apiextensions-apiserver/pkg/openapi/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "aggregator.go",
+        "construction.go",
+        "conversion.go",
+        "swagger_util.go",
+    ],
+    importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/openapi",
+    importpath = "k8s.io/apiextensions-apiserver/pkg/openapi",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/api/autoscaling/v1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/github.com/go-openapi/spec:go_default_library",
+        "//vendor/k8s.io/kube-openapi/pkg/aggregator:go_default_library",
+        "//vendor/k8s.io/kube-openapi/pkg/handler:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["conversion_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//vendor/github.com/go-openapi/spec:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

staging/src/k8s.io/apiextensions-apiserver/pkg/openapi/aggregator.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openapi
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/go-openapi/spec"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/kube-openapi/pkg/aggregator"
+	"k8s.io/kube-openapi/pkg/handler"
+)
+
+// AggregationManager is the interface between OpenAPI Aggregator service and a controller
+// that manages CRD openapi spec aggregation
+type AggregationManager interface {
+	// AddUpdateLocalAPIService allows adding/updating local API service with nil handler and
+	// nil Spec.Service. This function can be used for local dynamic OpenAPI spec aggregation
+	// management (e.g. CRD)
+	AddUpdateLocalAPIServiceSpec(name string, spec *spec.Swagger, etag string) error
+	RemoveAPIServiceSpec(apiServiceName string) error
+}
+
+type specAggregator struct {
+	// mutex protects all members of this struct.
+	rwMutex sync.RWMutex
+
+	// Map of API Services' OpenAPI specs by their name
+	openAPISpecs map[string]*openAPISpecInfo
+
+	// provided for dynamic OpenAPI spec
+	openAPIService          *handler.OpenAPIService
+	openAPIVersionedService *handler.OpenAPIService
+}
+
+var _ AggregationManager = &specAggregator{}
+
+// NewAggregationManager constructs a specAggregator from input openAPIService, openAPIVersionedService and
+// recorded static OpenAPI spec. The function returns an AggregationManager interface.
+func NewAggregationManager(openAPIService, openAPIVersionedService *handler.OpenAPIService, staticSpec *spec.Swagger) (AggregationManager, error) {
+	// openAPIVersionedService and deprecated openAPIService should be initialized together
+	if (openAPIService == nil) != (openAPIVersionedService == nil) {
+		return nil, fmt.Errorf("unexpected openapi service initialization error")
+	}
+	return &specAggregator{
+		openAPISpecs: map[string]*openAPISpecInfo{
+			"initial_static_spec": {
+				spec: staticSpec,
+			},
+		},
+		openAPIService:          openAPIService,
+		openAPIVersionedService: openAPIVersionedService,
+	}, nil
+}
+
+// openAPISpecInfo is used to store OpenAPI spec with its priority.
+// It can be used to sort specs with their priorities.
+type openAPISpecInfo struct {
+	// Name of a registered ApiService
+	name string
+
+	// Specification of this API Service. If null then the spec is not loaded yet.
+	spec *spec.Swagger
+	etag string
+}
+
+// buildOpenAPISpec aggregates all OpenAPI specs.  It is not thread-safe. The caller is responsible to hold proper locks.
+func (s *specAggregator) buildOpenAPISpec() (specToReturn *spec.Swagger, err error) {
+	specs := []openAPISpecInfo{}
+	for _, specInfo := range s.openAPISpecs {
+		if specInfo.spec == nil {
+			continue
+		}
+		specs = append(specs, *specInfo)
+	}
+	if len(specs) == 0 {
+		return &spec.Swagger{}, nil
+	}
+	for _, specInfo := range specs {
+		if specToReturn == nil {
+			specToReturn, err = aggregator.CloneSpec(specInfo.spec)
+			if err != nil {
+				return nil, err
+			}
+			continue
+		}
+		mergeSpecs(specToReturn, specInfo.spec)
+	}
+	// Add minimum required keys if missing, to properly serve the OpenAPI spec
+	// through apiextensions-apiserver HTTP handler. These keys will not be
+	// aggregated to top-level OpenAPI spec (only paths and definitions will).
+	// However these keys make the OpenAPI->proto serialization happy.
+	if specToReturn.Info == nil {
+		specToReturn.Info = &spec.Info{
+			InfoProps: spec.InfoProps{
+				Title: "Kubernetes",
+			},
+		}
+	}
+	if len(specToReturn.Swagger) == 0 {
+		specToReturn.Swagger = "2.0"
+	}
+	return specToReturn, nil
+}
+
+// updateOpenAPISpec aggregates all OpenAPI specs.  It is not thread-safe. The caller is responsible to hold proper locks.
+func (s *specAggregator) updateOpenAPISpec() error {
+	if s.openAPIService == nil || s.openAPIVersionedService == nil {
+		// openAPIVersionedService and deprecated openAPIService should be initialized together
+		if !(s.openAPIService == nil && s.openAPIVersionedService == nil) {
+			return fmt.Errorf("unexpected openapi service initialization error")
+		}
+		return nil
+	}
+	specToServe, err := s.buildOpenAPISpec()
+	if err != nil {
+		return err
+	}
+	// openAPIService.UpdateSpec and openAPIVersionedService.UpdateSpec read the same swagger spec
+	// serially and update their local caches separately. Both endpoints will have same spec in
+	// their caches if the caller is holding proper locks.
+	err = s.openAPIService.UpdateSpec(specToServe)
+	if err != nil {
+		return err
+	}
+	return s.openAPIVersionedService.UpdateSpec(specToServe)
+}
+
+// tryUpdatingServiceSpecs tries updating openAPISpecs map with specified specInfo, and keeps the map intact
+// if the update fails.
+func (s *specAggregator) tryUpdatingServiceSpecs(specInfo *openAPISpecInfo) error {
+	orgSpecInfo, exists := s.openAPISpecs[specInfo.name]
+	s.openAPISpecs[specInfo.name] = specInfo
+	if err := s.updateOpenAPISpec(); err != nil {
+		if exists {
+			s.openAPISpecs[specInfo.name] = orgSpecInfo
+		} else {
+			delete(s.openAPISpecs, specInfo.name)
+		}
+		return err
+	}
+	return nil
+}
+
+// tryDeleteServiceSpecs tries delete specified specInfo from openAPISpecs map, and keeps the map intact
+// if the update fails.
+func (s *specAggregator) tryDeleteServiceSpecs(apiServiceName string) error {
+	orgSpecInfo, exists := s.openAPISpecs[apiServiceName]
+	if !exists {
+		return nil
+	}
+	delete(s.openAPISpecs, apiServiceName)
+	if err := s.updateOpenAPISpec(); err != nil {
+		s.openAPISpecs[apiServiceName] = orgSpecInfo
+		return err
+	}
+	return nil
+}
+
+// AddUpdateLocalAPIService allows adding/updating local API service with nil handler and
+// nil Spec.Service. This function can be used for local dynamic OpenAPI spec aggregation
+// management (e.g. CRD)
+func (s *specAggregator) AddUpdateLocalAPIServiceSpec(name string, spec *spec.Swagger, etag string) error {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+
+	return s.tryUpdatingServiceSpecs(&openAPISpecInfo{
+		name: name,
+		spec: spec,
+		etag: etag,
+	})
+}
+
+// RemoveAPIServiceSpec removes an api service from OpenAPI aggregation. If it does not exist, no error is returned.
+// It is thread safe.
+func (s *specAggregator) RemoveAPIServiceSpec(apiServiceName string) error {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+
+	if _, existingService := s.openAPISpecs[apiServiceName]; !existingService {
+		return nil
+	}
+
+	return s.tryDeleteServiceSpecs(apiServiceName)
+}
+
+// mergeSpecs simply adds source openapi spec to dest and ignores any path/definition
+// conflicts because CRD openapi spec should not have conflict
+func mergeSpecs(dest, source *spec.Swagger) {
+	// Paths may be empty, due to [ACL constraints](http://goo.gl/8us55a#securityFiltering).
+	if source.Paths == nil {
+		// If Path is nil, none of the model defined in Definitions is used and we
+		// should not do anything.
+		// NOTE: this should not happen for CRD specs, because we automatically construct
+		// the Paths for CRD specs. We use utilruntime.HandleError to log this impossible
+		// case
+		utilruntime.HandleError(fmt.Errorf("unexpected CRD spec with empty Path: %v", *source))
+		return
+	}
+	if dest.Paths == nil {
+		dest.Paths = &spec.Paths{}
+	}
+	for k, v := range source.Definitions {
+		if dest.Definitions == nil {
+			dest.Definitions = spec.Definitions{}
+		}
+		dest.Definitions[k] = v
+	}
+	for k, v := range source.Paths.Paths {
+		// PathItem may be empty, due to [ACL constraints](http://goo.gl/8us55a#securityFiltering).
+		if dest.Paths.Paths == nil {
+			dest.Paths.Paths = map[string]spec.PathItem{}
+		}
+		dest.Paths.Paths[k] = v
+	}
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/openapi/conversion.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openapi
+
+import (
+	"encoding/json"
+
+	"github.com/go-openapi/spec"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+)
+
+// ConvertJSONSchemaPropsToOpenAPIv2Schema converts our internal OpenAPI v3 schema
+// (*apiextensions.JSONSchemaProps) to an OpenAPI v2 schema (*spec.Schema).
+// NOTE: we use versioned type (v1beta1) here so that we can properly marshal the object
+// using the JSON tags
+func ConvertJSONSchemaPropsToOpenAPIv2Schema(in *v1beta1.JSONSchemaProps) (*spec.Schema, error) {
+	if in == nil {
+		return nil, nil
+	}
+
+	// Marshal JSONSchemaProps into JSON and unmarshal the data into spec.Schema
+	data, err := json.Marshal(*in)
+	if err != nil {
+		return nil, err
+	}
+	out := new(spec.Schema)
+	if err := out.UnmarshalJSON(data); err != nil {
+		return nil, err
+	}
+	// Remove unsupported fields in OpenAPI v2
+	out.OneOf = nil
+	out.AnyOf = nil
+	out.Not = nil
+	return out, nil
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/openapi/swagger_util.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openapi
+
+import (
+	"crypto/sha512"
+	"encoding/json"
+	"fmt"
+
+	"github.com/go-openapi/spec"
+	autoscalingv1 "k8s.io/api/autoscaling/v1"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	deleteOptionsSchemaRef = "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.DeleteOptions"
+	objectMetaSchemaRef    = "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"
+	scaleSpecSchemaRef     = "#/definitions/io.k8s.api.autoscaling.v1.ScaleSpec"
+	scaleStatusSchemaRef   = "#/definitions/io.k8s.api.autoscaling.v1.ScaleStatus"
+)
+
+var swaggerTypeMetaDescriptions = metav1.TypeMeta{}.SwaggerDoc()
+var swaggerDeleteOptionsDescriptions = metav1.DeleteOptions{}.SwaggerDoc()
+var swaggerListDescriptions = metav1.List{}.SwaggerDoc()
+var swaggerListOptionsDescriptions = metav1.ListOptions{}.SwaggerDoc()
+var swaggerScaleDescriptions = autoscalingv1.Scale{}.SwaggerDoc()
+var swaggerScaleSpecDescriptions = autoscalingv1.ScaleSpec{}.SwaggerDoc()
+var swaggerScaleStatusDescriptions = autoscalingv1.ScaleStatus{}.SwaggerDoc()
+
+// calcSwaggerEtag calculates an etag of the OpenAPI swagger (spec)
+func calcSwaggerEtag(openAPISpec *spec.Swagger) (string, error) {
+	specBytes, err := json.MarshalIndent(openAPISpec, " ", " ")
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("\"%X\"", sha512.Sum512(specBytes)), nil
+}
+
+// pathParameters constructs the Parameter used by all paths in the CRD swagger (spec)
+func pathParameters() []spec.Parameter {
+	return []spec.Parameter{
+		*spec.QueryParam("pretty").
+			Typed("string", "").
+			UniqueValues().
+			WithDescription("If 'true', then the output is pretty printed."),
+	}
+}
+
+// addDeleteOperationParameters add the body&query parameters used by a delete operation
+func addDeleteOperationParameters(op *spec.Operation) *spec.Operation {
+	return op.
+		AddParam((&spec.Parameter{ParamProps: spec.ParamProps{Schema: spec.RefSchema(deleteOptionsSchemaRef)}}).
+			Named("body").
+			WithLocation("body").
+			AsRequired()).
+		AddParam(spec.QueryParam("dryRun").
+			Typed("string", "").
+			UniqueValues().
+			WithDescription(swaggerDeleteOptionsDescriptions["dryRun"])).
+		AddParam(spec.QueryParam("gracePeriodSeconds").
+			Typed("integer", "").
+			UniqueValues().
+			WithDescription(swaggerDeleteOptionsDescriptions["gracePeriodSeconds"])).
+		AddParam(spec.QueryParam("orphanDependents").
+			Typed("boolean", "").
+			UniqueValues().
+			WithDescription(swaggerDeleteOptionsDescriptions["orphanDependents"])).
+		AddParam(spec.QueryParam("propagationPolicy").
+			Typed("string", "").
+			UniqueValues().
+			WithDescription(swaggerDeleteOptionsDescriptions["propagationPolicy"]))
+}
+
+// addCollectionOperationParameters adds the query parameters used by list and deletecollection
+// operations
+func addCollectionOperationParameters(op *spec.Operation) *spec.Operation {
+	return op.
+		AddParam(spec.QueryParam("continue").
+			Typed("string", "").
+			UniqueValues().
+			WithDescription(swaggerListOptionsDescriptions["continue"])).
+		AddParam(spec.QueryParam("fieldSelector").
+			Typed("string", "").
+			UniqueValues().
+			WithDescription(swaggerListOptionsDescriptions["fieldSelector"])).
+		AddParam(spec.QueryParam("includeUninitialized").
+			Typed("boolean", "").
+			UniqueValues().
+			WithDescription(swaggerListOptionsDescriptions["includeUninitialized"])).
+		AddParam(spec.QueryParam("labelSelector").
+			Typed("string", "").
+			UniqueValues().
+			WithDescription(swaggerListOptionsDescriptions["labelSelector"])).
+		AddParam(spec.QueryParam("limit").
+			Typed("integer", "").
+			UniqueValues().
+			WithDescription(swaggerListOptionsDescriptions["limit"])).
+		AddParam(spec.QueryParam("resourceVersion").
+			Typed("string", "").
+			UniqueValues().
+			WithDescription(swaggerListOptionsDescriptions["resourceVersion"])).
+		AddParam(spec.QueryParam("timeoutSeconds").
+			Typed("integer", "").
+			UniqueValues().
+			WithDescription(swaggerListOptionsDescriptions["timeoutSeconds"])).
+		AddParam(spec.QueryParam("watch").
+			Typed("boolean", "").
+			UniqueValues().
+			WithDescription(swaggerListOptionsDescriptions["watch"]))
+}
+
+// okResponse constructs a 200 OK response with the input object schema reference
+func okResponse(ref string) *spec.Response {
+	return spec.NewResponse().
+		WithDescription("OK").
+		WithSchema(spec.RefSchema(ref))
+}
+
+// createdResponse constructs a 201 Created response with the input object schema reference
+func createdResponse(ref string) *spec.Response {
+	return spec.NewResponse().
+		WithDescription("Created").
+		WithSchema(spec.RefSchema(ref))
+}
+
+// acceptedResponse constructs a 202 Accepted response with the input object schema reference
+func acceptedResponse(ref string) *spec.Response {
+	return spec.NewResponse().
+		WithDescription("Accepted").
+		WithSchema(spec.RefSchema(ref))
+}
+
+// unauthorizedResponse constructs a 401 Unauthorized response
+func unauthorizedResponse() *spec.Response {
+	return spec.NewResponse().
+		WithDescription("Unauthorized")
+}
+
+// scaleSchema constructs the OpenAPI schema for io.k8s.api.autoscaling.v1.Scale objects
+// TODO(roycaihw): this is a hack to let apiExtension apiserver and generic kube-apiserver
+// to have the same io.k8s.api.autoscaling.v1.Scale definition, so that aggregator server won't
+// detect name conflict and create a duplicate io.k8s.api.autoscaling.v1.Scale_V2 schema
+// when aggregating the openapi spec. It would be better if apiExtension apiserver serves
+// identical definition through the same code path (using routes) as generic kube-apiserver.
+func scaleSchema() *spec.Schema {
+	s := new(spec.Schema).
+		WithDescription(swaggerScaleDescriptions[""]).
+		SetProperty("apiVersion", *spec.StringProperty().
+			WithDescription(swaggerTypeMetaDescriptions["apiVersion"])).
+		SetProperty("kind", *spec.StringProperty().
+			WithDescription(swaggerTypeMetaDescriptions["kind"])).
+		SetProperty("metadata", *spec.RefSchema(objectMetaSchemaRef).
+			WithDescription(swaggerScaleDescriptions["metadata"])).
+		SetProperty("spec", *spec.RefSchema(scaleSpecSchemaRef).
+			WithDescription(swaggerScaleDescriptions["spec"])).
+		SetProperty("status", *spec.RefSchema(scaleStatusSchemaRef).
+			WithDescription(swaggerScaleDescriptions["status"]))
+
+	s.AddExtension("x-kubernetes-group-version-kind", []map[string]string{
+		{
+			"group":   "autoscaling",
+			"kind":    "Scale",
+			"version": "v1",
+		},
+	})
+	return s
+}
+
+// scaleSchema constructs the OpenAPI schema for io.k8s.api.autoscaling.v1.ScaleSpec objects
+func scaleSpecSchema() *spec.Schema {
+	return new(spec.Schema).
+		WithDescription(swaggerScaleSpecDescriptions[""]).
+		SetProperty("replicas", *spec.Int32Property().
+			WithDescription(swaggerScaleSpecDescriptions["replicas"]))
+}
+
+// scaleSchema constructs the OpenAPI schema for io.k8s.api.autoscaling.v1.ScaleStatus objects
+func scaleStatusSchema() *spec.Schema {
+	return new(spec.Schema).
+		WithDescription(swaggerScaleStatusDescriptions[""]).
+		WithRequired("replicas").
+		SetProperty("replicas", *spec.Int32Property().
+			WithDescription(swaggerScaleStatusDescriptions["replicas"])).
+		SetProperty("selector", *spec.StringProperty().
+			WithDescription(swaggerScaleStatusDescriptions["selector"]))
+}
+
+// CustomResourceDefinitionOpenAPISpec constructs the OpenAPI spec (swagger) and calculates
+// etag for a given CustomResourceDefinitionSpec.
+// NOTE: in apiserver we general operates on internal types. We are using versioned (v1beta1)
+// validation schema here because we need the json tags to properly marshal the object to
+// JSON.
+func CustomResourceDefinitionOpenAPISpec(crdSpec *apiextensions.CustomResourceDefinitionSpec, version string, validationSchema *v1beta1.CustomResourceValidation) (*spec.Swagger, string, error) {
+	schema := &spec.Schema{}
+	if validationSchema != nil && validationSchema.OpenAPIV3Schema != nil {
+		var err error
+		schema, err = ConvertJSONSchemaPropsToOpenAPIv2Schema(validationSchema.OpenAPIV3Schema)
+		if err != nil {
+			return nil, "", err
+		}
+	}
+	crdSwaggerConstructor, err := NewSwaggerConstructor(schema, crdSpec, version)
+	if err != nil {
+		return nil, "", err
+	}
+	crdOpenAPISpec := crdSwaggerConstructor.ConstructCRDOpenAPISpec()
+	etag, err := calcSwaggerEtag(crdOpenAPISpec)
+	if err != nil {
+		return nil, "", err
+	}
+	return crdOpenAPISpec, etag, nil
+}

staging/src/k8s.io/apiserver/pkg/server/BUILD

@@ -116,6 +116,7 @@ go_library(
         "//vendor/k8s.io/klog:go_default_library",
         "//vendor/k8s.io/kube-openapi/pkg/builder:go_default_library",
         "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library",
+        "//vendor/k8s.io/kube-openapi/pkg/handler:go_default_library",
         "//vendor/k8s.io/kube-openapi/pkg/util:go_default_library",
         "//vendor/k8s.io/kube-openapi/pkg/util/proto:go_default_library",
     ],

staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -25,7 +25,8 @@ import (
 	"time"
 
 	systemd "github.com/coreos/go-systemd/daemon"
-	"github.com/emicklei/go-restful-swagger12"
+	swagger "github.com/emicklei/go-restful-swagger12"
+	"github.com/go-openapi/spec"
 	"k8s.io/klog"
 
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -47,6 +48,7 @@ import (
 	restclient "k8s.io/client-go/rest"
 	openapibuilder "k8s.io/kube-openapi/pkg/builder"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
+	"k8s.io/kube-openapi/pkg/handler"
 	openapiutil "k8s.io/kube-openapi/pkg/util"
 	openapiproto "k8s.io/kube-openapi/pkg/util/proto"
 )
@@ -124,6 +126,11 @@ type GenericAPIServer struct {
 	swaggerConfig *swagger.Config
 	openAPIConfig *openapicommon.Config
 
+	// Expose the registered OpenAPI Services and built static OpenAPI spec if openAPIConfig is non-nil
+	OpenAPIService          *handler.OpenAPIService // for endpoint /swagger.json
+	OpenAPIVersionedService *handler.OpenAPIService // for endpoint /openapi/v2
+	StaticOpenAPISpec       *spec.Swagger
+
 	// PostStartHooks are each called after the server has started listening, in a separate go func for each
 	// with no guarantee of ordering between them.  The map key is a name used for error reporting.
 	// It may kill the process with a panic if it wishes to by returning an error.
@@ -240,7 +247,7 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
 		routes.Swagger{Config: s.swaggerConfig}.Install(s.Handler.GoRestfulContainer)
 	}
 	if s.openAPIConfig != nil {
-		routes.OpenAPI{
+		s.OpenAPIService, s.OpenAPIVersionedService, s.StaticOpenAPISpec = routes.OpenAPI{
 			Config: s.openAPIConfig,
 		}.Install(s.Handler.GoRestfulContainer, s.Handler.NonGoRestfulMux)
 	}

staging/src/k8s.io/apiserver/pkg/server/routes/BUILD

@@ -32,8 +32,10 @@ go_library(
         "//vendor/github.com/elazarl/go-bindata-assetfs:go_default_library",
         "//vendor/github.com/emicklei/go-restful:go_default_library",
         "//vendor/github.com/emicklei/go-restful-swagger12:go_default_library",
+        "//vendor/github.com/go-openapi/spec:go_default_library",
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
+        "//vendor/k8s.io/kube-openapi/pkg/builder:go_default_library",
         "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library",
         "//vendor/k8s.io/kube-openapi/pkg/handler:go_default_library",
     ],

staging/src/k8s.io/apiserver/pkg/server/routes/openapi.go

@@ -18,9 +18,11 @@ package routes
 
 import (
 	restful "github.com/emicklei/go-restful"
+	"github.com/go-openapi/spec"
 	"k8s.io/klog"
 
 	"k8s.io/apiserver/pkg/server/mux"
+	"k8s.io/kube-openapi/pkg/builder"
 	"k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kube-openapi/pkg/handler"
 )
@@ -30,17 +32,27 @@ type OpenAPI struct {
 	Config *common.Config
 }
 
-// Install adds the SwaggerUI webservice to the given mux.
-func (oa OpenAPI) Install(c *restful.Container, mux *mux.PathRecorderMux) {
+// Install adds the SwaggerUI webservice to the given mux. This function returns
+// the built static OpenAPI spec and the registered OpenAPI services to allow
+// further OpenAPI spec aggregation.
+func (oa OpenAPI) Install(c *restful.Container, mux *mux.PathRecorderMux) (openAPIService, openAPIVersionedService *handler.OpenAPIService, spec *spec.Swagger) {
+	var err error
+	// Record the static OpenAPI spec to allow further OpenAPI spec aggregation
+	// with this static spec on the registered OpenAPI services
+	spec, err = builder.BuildOpenAPISpec(c.RegisteredWebServices(), oa.Config)
+	if err != nil {
+		klog.Fatalf("Failed to build open api spec for root: %v", err)
+	}
 	// NOTE: [DEPRECATION] We will announce deprecation for format-separated endpoints for OpenAPI spec,
 	// and switch to a single /openapi/v2 endpoint in Kubernetes 1.10. The design doc and deprecation process
 	// are tracked at: https://docs.google.com/document/d/19lEqE9lc4yHJ3WJAJxS_G7TcORIJXGHyq3wpwcH28nU.
-	_, err := handler.BuildAndRegisterOpenAPIService("/swagger.json", c.RegisteredWebServices(), oa.Config, mux)
+	openAPIService, err = handler.RegisterOpenAPIService(spec, "/swagger.json", mux)
 	if err != nil {
 		klog.Fatalf("Failed to register open api spec for root: %v", err)
 	}
-	_, err = handler.BuildAndRegisterOpenAPIVersionedService("/openapi/v2", c.RegisteredWebServices(), oa.Config, mux)
+	openAPIVersionedService, err = handler.RegisterOpenAPIVersionedService(spec, "/openapi/v2", mux)
 	if err != nil {
 		klog.Fatalf("Failed to register versioned open api spec for root: %v", err)
 	}
+	return openAPIService, openAPIVersionedService, spec
 }

staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator.go

@@ -53,12 +53,19 @@ type specAggregator struct {
 	// provided for dynamic OpenAPI spec
 	openAPIService          *handler.OpenAPIService
 	openAPIVersionedService *handler.OpenAPIService
+
+	// initialized is set to be true at the end of startup. All local specs
+	// must be registered before initialized is set, we panic otherwise.
+	initialized bool
 }
 
 var _ AggregationManager = &specAggregator{}
 
 // This function is not thread safe as it only being called on startup.
 func (s *specAggregator) addLocalSpec(spec *spec.Swagger, localHandler http.Handler, name, etag string) {
+	if s.initialized {
+		panic("Local spec must not be added after startup")
+	}
 	localAPIService := apiregistration.APIService{}
 	localAPIService.Name = name
 	s.openAPISpecs[name] = &openAPISpecInfo{
@@ -69,6 +76,17 @@ func (s *specAggregator) addLocalSpec(spec *spec.Swagger, localHandler http.Hand
 	}
 }
 
+// GetAPIServicesName returns the names of APIServices recorded in specAggregator.openAPISpecs.
+// We use this function to pass the names of local APIServices to the controller in this package,
+// so that the controller can periodically sync the OpenAPI spec from delegation API servers.
+func (s *specAggregator) GetAPIServiceNames() []string {
+	names := make([]string, len(s.openAPISpecs))
+	for key := range s.openAPISpecs {
+		names = append(names, key)
+	}
+	return names
+}
+
 // BuildAndRegisterAggregator registered OpenAPI aggregator handler. This function is not thread safe as it only being called on startup.
 func BuildAndRegisterAggregator(downloader *Downloader, delegationTarget server.DelegationTarget, webServices []*restful.WebService,
 	config *common.Config, pathHandler common.PathHandler) (AggregationManager, error) {
@@ -124,6 +142,9 @@ func BuildAndRegisterAggregator(downloader *Downloader, delegationTarget server.
 		return nil, err
 	}
 
+	// We set initialized to be true to forbid any future local spec addition
+	s.initialized = true
+
 	return s, nil
 }
 

staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/controller.go

@@ -49,6 +49,9 @@ type AggregationManager interface {
 	UpdateAPIServiceSpec(apiServiceName string, spec *spec.Swagger, etag string) error
 	RemoveAPIServiceSpec(apiServiceName string) error
 	GetAPIServiceInfo(apiServiceName string) (handler http.Handler, etag string, exists bool)
+
+	// GetAPIServicesName returns the names of APIServices recorded in AggregationManager.
+	GetAPIServiceNames() []string
 }
 
 // AggregationController periodically check for changes in OpenAPI specs of APIServices and update/remove
@@ -72,6 +75,18 @@ func NewAggregationController(downloader *Downloader, openAPIAggregationManager
 	}
 
 	c.syncHandler = c.sync
+	// During initialization, openAPIAggregationManager only has record of local APIServices. There must be
+	// no aggregated APIService recorded, because aggregated APIServices only get added to openAPIAggregationManager
+	// by calling AggregationController.AddAPIService or AggregationController.UpdateAPIService after the
+	// controller is initialized.
+	// Here we add delegation target API services to queue, to periodically sync dynamic OpenAPI spec from
+	// delegation target.
+	// NOTE: openAPIAggregationManager.GetAPIServiceNames() will also return the APIService of non-name spec
+	// for aggregator, which has no http.Handler. The first time sync (when popping off from queue) for
+	// this APIService will be a no-op, and the controller will drop the APIService from queue.
+	for _, name := range openAPIAggregationManager.GetAPIServiceNames() {
+		c.queue.AddAfter(name, time.Second)
+	}
 
 	return c
 }

test/e2e/apimachinery/custom_resource_definition.go

@@ -17,16 +17,22 @@ limitations under the License.
 package apimachinery
 
 import (
+	"fmt"
+	"strings"
+	"time"
+
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apiextensions-apiserver/test/integration/fixtures"
 	utilversion "k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
 )
 
 var crdVersion = utilversion.MustParseSemantic("v1.7.0")
+var crdOpenAPIVersion = utilversion.MustParseSemantic("v1.13.0")
 
 var _ = SIGDescribe("CustomResourceDefinition resources", func() {
 
@@ -67,5 +73,58 @@ var _ = SIGDescribe("CustomResourceDefinition resources", func() {
 				}
 			}()
 		})
+
+		It("has OpenAPI spec served with CRD Validation chema", func() {
+			framework.SkipUnlessServerVersionGTE(crdOpenAPIVersion, f.ClientSet.Discovery())
+
+			config, err := framework.LoadConfig()
+			if err != nil {
+				framework.Failf("failed to load config: %v", err)
+			}
+
+			apiExtensionClient, err := clientset.NewForConfig(config)
+			if err != nil {
+				framework.Failf("failed to initialize apiExtensionClient: %v", err)
+			}
+
+			randomDefinition := fixtures.NewRandomNameCustomResourceDefinition(v1beta1.ClusterScoped)
+
+			//create CRD and waits for the resource to be recognized and available.
+			randomDefinition, err = fixtures.CreateNewCustomResourceDefinition(randomDefinition, apiExtensionClient, f.DynamicClient)
+			if err != nil {
+				framework.Failf("failed to create CustomResourceDefinition: %v", err)
+			}
+
+			// TODO(roycaihw): think about tweaking feature gates in e2e test (is it possible/easy
+			// to do so?) and have CRD use top-level/per-version schema
+			// Also need to test NamespaceScoped CRDs
+
+			// We use a wait.Poll block here because the kube-aggregator openapi
+			// controller takes time to rotate the queue and resync apiextensions-apiserver's spec
+			if err := wait.Poll(5*time.Second, 120*time.Second, func() (bool, error) {
+				data, err := f.ClientSet.CoreV1().RESTClient().Get().
+					AbsPath("/swagger.json").
+					DoRaw()
+
+				if err != nil {
+					return false, err
+				}
+				// TODO(roycaihw): verify more Paths and List Definitions, also for multiple versions
+				baseDefinition := fmt.Sprintf("%s.%s.%s", randomDefinition.Spec.Group, randomDefinition.Spec.Version, randomDefinition.Spec.Names.Kind)
+				basePath := fmt.Sprintf("/apis/%s/%s/%s", randomDefinition.Spec.Group, randomDefinition.Spec.Version, randomDefinition.Spec.Names.Plural)
+				return strings.Contains(string(data), basePath) &&
+					strings.Contains(string(data), baseDefinition), nil
+			}); err != nil {
+				framework.Failf("failed to wait for apiserver to serve openapi spec for registered CRD: %v", err)
+			}
+
+			defer func() {
+				err = fixtures.DeleteCustomResourceDefinition(randomDefinition, apiExtensionClient)
+				if err != nil {
+					framework.Failf("failed to delete CustomResourceDefinition: %v", err)
+				}
+			}()
+		})
+
 	})
 })

test/integration/master/BUILD

@@ -66,6 +66,7 @@ go_test(
         "//test/integration/framework:go_default_library",
         "//test/utils:go_default_library",
         "//vendor/github.com/evanphx/json-patch:go_default_library",
+        "//vendor/github.com/go-openapi/spec:go_default_library",
         "//vendor/sigs.k8s.io/yaml:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:android": [

test/integration/master/crd_test.go

@@ -18,9 +18,12 @@ package master
 
 import (
 	"encoding/json"
+	"fmt"
 	"testing"
 	"time"
 
+	"github.com/go-openapi/spec"
+
 	admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1"
 	networkingv1 "k8s.io/api/networking/v1"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
@@ -272,6 +275,91 @@ func TestCRD(t *testing.T) {
 	}
 }
 
+func TestCRDOpenAPI(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.Initializers, true)()
+	result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
+	defer result.TearDownFn()
+	kubeclient, err := kubernetes.NewForConfig(result.ClientConfig)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	t.Logf("Trying to create a custom resource without conflict")
+	crd := &apiextensionsv1beta1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "foos.cr.bar.com",
+		},
+		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+			Group:   "cr.bar.com",
+			Version: "v1",
+			Scope:   apiextensionsv1beta1.NamespaceScoped,
+			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+				Plural: "foos",
+				Kind:   "Foo",
+			},
+			Validation: &apiextensionsv1beta1.CustomResourceValidation{
+				OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
+					Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
+						"foo": {Type: "string"},
+					},
+				},
+			},
+		},
+	}
+	etcd.CreateTestCRDs(t, apiextensionsclient, false, crd)
+	waitForSpec := func(expectedType string) {
+		t.Logf(`Waiting for {properties: {"foo": {"type":"%s"}}} to show up in schema`, expectedType)
+		lastMsg := ""
+		if err := wait.PollImmediate(500*time.Millisecond, 120*time.Second, func() (bool, error) {
+			lastMsg = ""
+			bs, err := kubeclient.RESTClient().Get().AbsPath("openapi", "v2").DoRaw()
+			if err != nil {
+				return false, err
+			}
+			spec := spec.Swagger{}
+			if err := json.Unmarshal(bs, &spec); err != nil {
+				return false, err
+			}
+			if spec.SwaggerProps.Paths == nil {
+				lastMsg = "spec.SwaggerProps.Paths is nil"
+				return false, nil
+			}
+			d, ok := spec.SwaggerProps.Definitions["cr.bar.com.v1.Foo"]
+			if !ok {
+				lastMsg = `spec.SwaggerProps.Definitions["cr.bar.com.v1.Foo"] not found`
+				return false, nil
+			}
+			p, ok := d.Properties["foo"]
+			if !ok {
+				lastMsg = `spec.SwaggerProps.Definitions["cr.bar.com.v1.Foo"].Properties["foo"] not found`
+				return false, nil
+			}
+			if !p.Type.Contains(expectedType) {
+				lastMsg = fmt.Sprintf(`spec.SwaggerProps.Definitions["cr.bar.com.v1.Foo"].Properties["foo"].Type should be %q, but got: %q`, expectedType, p.Type)
+				return false, nil
+			}
+			return true, nil
+		}); err != nil {
+			t.Fatalf("Failed to see %s OpenAPI spec in discovery: %v, last message: %s", crd.Name, err, lastMsg)
+		}
+	}
+	waitForSpec("string")
+	crd, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	prop := crd.Spec.Validation.OpenAPIV3Schema.Properties["foo"]
+	prop.Type = "boolean"
+	crd.Spec.Validation.OpenAPIV3Schema.Properties["foo"] = prop
+	if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd); err != nil {
+		t.Fatal(err)
+	}
+	waitForSpec("boolean")
+}
+
 type Foo struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`