Merge pull request #67006 from mbohlool/crd_webhook_conversion

cmd/kube-apiserver/app/apiextensions.go

@@ -28,6 +28,7 @@ import (
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/apiserver/pkg/util/webhook"
 	kubeexternalinformers "k8s.io/client-go/informers"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 )
@@ -38,6 +39,8 @@ func createAPIExtensionsConfig(
 	pluginInitializers []admission.PluginInitializer,
 	commandOptions *options.ServerRunOptions,
 	masterCount int,
+	serviceResolver webhook.ServiceResolver,
+	authResolverWrapper webhook.AuthenticationInfoResolverWrapper,
 ) (*apiextensionsapiserver.Config, error) {
 	// make a shallow copy to let us twiddle a few things
 	// most of the config actually remains the same.  We only need to mess with a couple items related to the particulars of the apiextensions
@@ -74,6 +77,8 @@ func createAPIExtensionsConfig(
 		ExtraConfig: apiextensionsapiserver.ExtraConfig{
 			CRDRESTOptionsGetter: apiextensionsoptions.NewCRDRESTOptionsGetter(etcdOptions),
 			MasterCount:          masterCount,
+			AuthResolverWrapper:  authResolverWrapper,
+			ServiceResolver:      serviceResolver,
 		},
 	}
 

cmd/kube-apiserver/app/server.go

@@ -165,7 +165,8 @@ func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan
 	}
 
 	// If additional API servers are added, they should be gated.
-	apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount)
+	apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount,
+		serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(proxyTransport, kubeAPIServerConfig.GenericConfig.LoopbackClientConfig))
 	if err != nil {
 		return nil, err
 	}

staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json

@@ -2343,6 +2343,10 @@
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
 		{
+			"ImportPath": "k8s.io/apimachinery/pkg/util/uuid",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/validation",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
@@ -2451,6 +2455,10 @@
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
 		{
+			"ImportPath": "k8s.io/apiserver/pkg/util/proxy",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
 			"ImportPath": "k8s.io/apiserver/pkg/util/webhook",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
@@ -2475,6 +2483,10 @@
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
 		{
+			"ImportPath": "k8s.io/client-go/listers/core/v1",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
 			"ImportPath": "k8s.io/client-go/rest",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD

@@ -67,6 +67,7 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library",
         "//staging/src/k8s.io/client-go/scale:go_default_library",
         "//staging/src/k8s.io/client-go/scale/scheme/autoscalingv1:go_default_library",
         "//staging/src/k8s.io/client-go/tools/cache:go_default_library",

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/apiserver/pkg/registry/rest"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
+	"k8s.io/apiserver/pkg/util/webhook"
 
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install"
@@ -78,6 +79,11 @@ type ExtraConfig struct {
 	// MasterCount is used to detect whether cluster is HA, and if it is
 	// the CRD Establishing will be hold by 5 seconds.
 	MasterCount int
+
+	// ServiceResolver is used in CR webhook converters to resolve webhook's service names
+	ServiceResolver webhook.ServiceResolver
+	// AuthResolverWrapper is used in CR webhook converters
+	AuthResolverWrapper webhook.AuthenticationInfoResolverWrapper
 }
 
 type Config struct {
@@ -167,7 +173,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 		delegate:  delegateHandler,
 	}
 	establishingController := establish.NewEstablishingController(s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions(), crdClient.Apiextensions())
-	crdHandler := NewCustomResourceDefinitionHandler(
+	crdHandler, err := NewCustomResourceDefinitionHandler(
 		versionDiscoveryHandler,
 		groupDiscoveryHandler,
 		s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions(),
@@ -175,8 +181,13 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 		c.ExtraConfig.CRDRESTOptionsGetter,
 		c.GenericConfig.AdmissionControl,
 		establishingController,
+		c.ExtraConfig.ServiceResolver,
+		c.ExtraConfig.AuthResolverWrapper,
 		c.ExtraConfig.MasterCount,
 	)
+	if err != nil {
+		return nil, err
+	}
 	s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", crdHandler)
 	s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/apis/", crdHandler)
 

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD

@@ -5,15 +5,23 @@ go_library(
     srcs = [
         "converter.go",
         "nop_converter.go",
+        "webhook_converter.go",
     ],
     importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion",
     importpath = "k8s.io/apiextensions-apiserver/pkg/apiserver/conversion",
     visibility = ["//visibility:public"],
     deps = [
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
     ],
 )
 

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go

@@ -20,39 +20,74 @@ import (
 	"fmt"
 
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/apiserver/pkg/util/webhook"
 )
 
-// NewCRDConverter returns a new CRD converter based on the conversion settings in crd object.
-func NewCRDConverter(crd *apiextensions.CustomResourceDefinition) (safe, unsafe runtime.ObjectConvertor) {
+// CRConverterFactory is the factory for all CR converters.
+type CRConverterFactory struct {
+	// webhookConverterFactory is the factory for webhook converters.
+	// This field should not be used if CustomResourceWebhookConversion feature is disabled.
+	webhookConverterFactory *webhookConverterFactory
+}
+
+// NewCRConverterFactory creates a new CRConverterFactory
+func NewCRConverterFactory(serviceResolver webhook.ServiceResolver, authResolverWrapper webhook.AuthenticationInfoResolverWrapper) (*CRConverterFactory, error) {
+	converterFactory := &CRConverterFactory{}
+	if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) {
+		webhookConverterFactory, err := newWebhookConverterFactory(serviceResolver, authResolverWrapper)
+		if err != nil {
+			return nil, err
+		}
+		converterFactory.webhookConverterFactory = webhookConverterFactory
+	}
+	return converterFactory, nil
+}
+
+// NewConverter returns a new CR converter based on the conversion settings in crd object.
+func (m *CRConverterFactory) NewConverter(crd *apiextensions.CustomResourceDefinition) (safe, unsafe runtime.ObjectConvertor, err error) {
 	validVersions := map[schema.GroupVersion]bool{}
 	for _, version := range crd.Spec.Versions {
 		validVersions[schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name}] = true
 	}
 
-	// The only converter right now is nopConverter. More converters will be returned based on the
-	// CRD object when they introduced.
-	unsafe = &crdConverter{
-		clusterScoped: crd.Spec.Scope == apiextensions.ClusterScoped,
-		delegate: &nopConverter{
-			validVersions: validVersions,
-		},
+	switch crd.Spec.Conversion.Strategy {
+	case apiextensions.NoneConverter:
+		unsafe = &crConverter{
+			clusterScoped: crd.Spec.Scope == apiextensions.ClusterScoped,
+			delegate: &nopConverter{
+				validVersions: validVersions,
+			},
+		}
+		return &safeConverterWrapper{unsafe}, unsafe, nil
+	case apiextensions.WebhookConverter:
+		if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) {
+			return nil, nil, fmt.Errorf("webhook conversion is disabled on this cluster")
+		}
+		unsafe, err := m.webhookConverterFactory.NewWebhookConverter(validVersions, crd)
+		if err != nil {
+			return nil, nil, err
+		}
+		return &safeConverterWrapper{unsafe}, unsafe, nil
 	}
-	return &safeConverterWrapper{unsafe}, unsafe
+
+	return nil, nil, fmt.Errorf("unknown conversion strategy %q for CRD %s", crd.Spec.Conversion.Strategy, crd.Name)
 }
 
-var _ runtime.ObjectConvertor = &crdConverter{}
+var _ runtime.ObjectConvertor = &crConverter{}
 
-// crdConverter extends the delegate with generic CRD conversion behaviour. The delegate will implement the
+// crConverter extends the delegate with generic CR conversion behaviour. The delegate will implement the
 // user defined conversion strategy given in the CustomResourceDefinition.
-type crdConverter struct {
+type crConverter struct {
 	delegate      runtime.ObjectConvertor
 	clusterScoped bool
 }
 
-func (c *crdConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, value string) (string, string, error) {
+func (c *crConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, value string) (string, string, error) {
 	// We currently only support metadata.namespace and metadata.name.
 	switch {
 	case label == "metadata.name":
@@ -64,12 +99,12 @@ func (c *crdConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, val
 	}
 }
 
-func (c *crdConverter) Convert(in, out, context interface{}) error {
+func (c *crConverter) Convert(in, out, context interface{}) error {
 	return c.delegate.Convert(in, out, context)
 }
 
 // ConvertToVersion converts in object to the given gvk in place and returns the same `in` object.
-func (c *crdConverter) ConvertToVersion(in runtime.Object, target runtime.GroupVersioner) (runtime.Object, error) {
+func (c *crConverter) ConvertToVersion(in runtime.Object, target runtime.GroupVersioner) (runtime.Object, error) {
 	// Run the converter on the list items instead of list itself
 	if list, ok := in.(*unstructured.UnstructuredList); ok {
 		for i := range list.Items {

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/nop_converter.go

@@ -49,11 +49,11 @@ func (c *nopConverter) Convert(in, out, context interface{}) error {
 
 	outGVK := unstructOut.GroupVersionKind()
 	if !c.validVersions[outGVK.GroupVersion()] {
-		return fmt.Errorf("request to convert CRD from an invalid group/version: %s", outGVK.String())
+		return fmt.Errorf("request to convert CR from an invalid group/version: %s", outGVK.String())
 	}
 	inGVK := unstructIn.GroupVersionKind()
 	if !c.validVersions[inGVK.GroupVersion()] {
-		return fmt.Errorf("request to convert CRD to an invalid group/version: %s", inGVK.String())
+		return fmt.Errorf("request to convert CR to an invalid group/version: %s", inGVK.String())
 	}
 
 	unstructOut.SetUnstructuredContent(unstructIn.UnstructuredContent())
@@ -72,7 +72,7 @@ func (c *nopConverter) ConvertToVersion(in runtime.Object, target runtime.GroupV
 		return nil, fmt.Errorf("%v is unstructured and is not suitable for converting to %q", kind, target)
 	}
 	if !c.validVersions[gvk.GroupVersion()] {
-		return nil, fmt.Errorf("request to convert CRD to an invalid group/version: %s", gvk.String())
+		return nil, fmt.Errorf("request to convert CR to an invalid group/version: %s", gvk.String())
 	}
 	in.GetObjectKind().SetGroupVersionKind(gvk)
 	return in, nil

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go

@@ -67,6 +67,7 @@ import (
 	apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features"
 	"k8s.io/apiextensions-apiserver/pkg/registry/customresource"
 	"k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor"
+	"k8s.io/apiserver/pkg/util/webhook"
 )
 
 // crdHandler serves the `/apis` endpoint.
@@ -93,6 +94,8 @@ type crdHandler struct {
 	// MasterCount is used to implement sleep to improve
 	// CRD establishing process for HA clusters.
 	masterCount int
+
+	converterFactory *conversion.CRConverterFactory
 }
 
 // crdInfo stores enough information to serve the storage for the custom resource
@@ -129,7 +132,9 @@ func NewCustomResourceDefinitionHandler(
 	restOptionsGetter generic.RESTOptionsGetter,
 	admission admission.Interface,
 	establishingController *establish.EstablishingController,
-	masterCount int) *crdHandler {
+	serviceResolver webhook.ServiceResolver,
+	authResolverWrapper webhook.AuthenticationInfoResolverWrapper,
+	masterCount int) (*crdHandler, error) {
 	ret := &crdHandler{
 		versionDiscoveryHandler: versionDiscoveryHandler,
 		groupDiscoveryHandler:   groupDiscoveryHandler,
@@ -147,10 +152,15 @@ func NewCustomResourceDefinitionHandler(
 			ret.removeDeadStorage()
 		},
 	})
+	crConverterFactory, err := conversion.NewCRConverterFactory(serviceResolver, authResolverWrapper)
+	if err != nil {
+		return nil, err
+	}
+	ret.converterFactory = crConverterFactory
 
 	ret.customStorage.Store(crdStorageMap{})
 
-	return ret
+	return ret, nil
 }
 
 func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
@@ -433,7 +443,10 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource
 	scaleScopes := map[string]handlers.RequestScope{}
 
 	for _, v := range crd.Spec.Versions {
-		safeConverter, unsafeConverter := conversion.NewCRDConverter(crd)
+		safeConverter, unsafeConverter, err := r.converterFactory.NewConverter(crd)
+		if err != nil {
+			return nil, err
+		}
 		// In addition to Unstructured objects (Custom Resources), we also may sometimes need to
 		// decode unversioned Options objects, so we delegate to parameterScheme for such types.
 		parameterScheme := runtime.NewScheme()

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler_test.go

@@ -79,14 +79,24 @@ func TestConvertFieldLabel(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 
-			crd := apiextensions.CustomResourceDefinition{}
+			crd := apiextensions.CustomResourceDefinition{
+				Spec: apiextensions.CustomResourceDefinitionSpec{
+					Conversion: &apiextensions.CustomResourceConversion{
+						Strategy: "None",
+					},
+				},
+			}
 
 			if test.clusterScoped {
 				crd.Spec.Scope = apiextensions.ClusterScoped
 			} else {
 				crd.Spec.Scope = apiextensions.NamespaceScoped
 			}
-			_, c := conversion.NewCRDConverter(&crd)
+			f, err := conversion.NewCRConverterFactory(nil, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			_, c, err := f.NewConverter(&crd)
 
 			label, value, err := c.ConvertFieldLabel(schema.GroupVersionKind{}, test.label, "value")
 			if e, a := test.expectError, err != nil; e != a {

staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/BUILD

@@ -14,6 +14,9 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/proxy:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library",
+        "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
     ],
 )

staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"net/url"
 
 	"github.com/spf13/pflag"
 
@@ -30,6 +31,9 @@ import (
 	genericregistry "k8s.io/apiserver/pkg/registry/generic"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
+	"k8s.io/apiserver/pkg/util/proxy"
+	"k8s.io/apiserver/pkg/util/webhook"
+	"k8s.io/client-go/listers/core/v1"
 )
 
 const defaultEtcdPathPrefix = "/registry/apiextensions.kubernetes.io"
@@ -94,6 +98,8 @@ func (o CustomResourceDefinitionsServerOptions) Config() (*apiserver.Config, err
 		GenericConfig: serverConfig,
 		ExtraConfig: apiserver.ExtraConfig{
 			CRDRESTOptionsGetter: NewCRDRESTOptionsGetter(*o.RecommendedOptions.Etcd),
+			ServiceResolver:      &serviceResolver{serverConfig.SharedInformerFactory.Core().V1().Services().Lister()},
+			AuthResolverWrapper:  webhook.NewDefaultAuthenticationInfoResolverWrapper(nil, serverConfig.LoopbackClientConfig),
 		},
 	}
 	return config, nil
@@ -114,3 +120,11 @@ func NewCRDRESTOptionsGetter(etcdOptions genericoptions.EtcdOptions) genericregi
 
 	return ret
 }
+
+type serviceResolver struct {
+	services v1.ServiceLister
+}
+
+func (r *serviceResolver) ResolveEndpoint(namespace, name string) (*url.URL, error) {
+	return proxy.ResolveCluster(r.services, namespace, name)
+}

staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go

@@ -76,8 +76,8 @@ func newNamespacedCustomResourceClient(ns string, client dynamic.Interface, crd
 	return newNamespacedCustomResourceVersionedClient(ns, client, crd, crd.Spec.Versions[0].Name)
 }
 
-// updateCustomResourceDefinitionWithRetry updates a CRD, retrying up to 5 times on version conflict errors.
-func updateCustomResourceDefinitionWithRetry(client clientset.Interface, name string, update func(*apiextensionsv1beta1.CustomResourceDefinition)) (*apiextensionsv1beta1.CustomResourceDefinition, error) {
+// UpdateCustomResourceDefinitionWithRetry updates a CRD, retrying up to 5 times on version conflict errors.
+func UpdateCustomResourceDefinitionWithRetry(client clientset.Interface, name string, update func(*apiextensionsv1beta1.CustomResourceDefinition)) (*apiextensionsv1beta1.CustomResourceDefinition, error) {
 	for i := 0; i < 5; i++ {
 		crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
 		if err != nil {

staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go

@@ -445,7 +445,7 @@ func TestCRValidationOnCRDUpdate(t *testing.T) {
 			}
 
 			// update the CRD to a less stricter schema
-			_, err = updateCustomResourceDefinitionWithRetry(apiExtensionClient, "noxus.mygroup.example.com", func(crd *apiextensionsv1beta1.CustomResourceDefinition) {
+			_, err = UpdateCustomResourceDefinitionWithRetry(apiExtensionClient, "noxus.mygroup.example.com", func(crd *apiextensionsv1beta1.CustomResourceDefinition) {
 				validationSchema, err := getSchemaForVersion(crd, v.Name)
 				if err != nil {
 					t.Fatal(err)

staging/src/k8s.io/apiextensions-apiserver/test/integration/versioning_test.go

@@ -64,7 +64,7 @@ func TestInternalVersionIsHandlerVersion(t *testing.T) {
 
 	// update validation via update because the cache priming in CreateNewCustomResourceDefinition will fail otherwise
 	t.Logf("Updating CRD to validate apiVersion")
-	noxuDefinition, err = updateCustomResourceDefinitionWithRetry(apiExtensionClient, noxuDefinition.Name, func(crd *apiextensionsv1beta1.CustomResourceDefinition) {
+	noxuDefinition, err = UpdateCustomResourceDefinitionWithRetry(apiExtensionClient, noxuDefinition.Name, func(crd *apiextensionsv1beta1.CustomResourceDefinition) {
 		crd.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{
 			OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
 				Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{

staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go

@@ -164,6 +164,7 @@ func (a *Webhook) Dispatch(attr admission.Attributes) error {
 		return admission.NewForbidden(attr, fmt.Errorf("not yet ready to handle request"))
 	}
 	hooks := a.hookSource.Webhooks()
+	// TODO: Figure out if adding one second timeout make sense here.
 	ctx := context.TODO()
 
 	var relevantHooks []*v1beta1.Webhook

test/e2e/apimachinery/BUILD

@@ -11,6 +11,7 @@ go_library(
         "aggregator.go",
         "certs.go",
         "chunking.go",
+        "crd_conversion_webhook.go",
         "crd_watch.go",
         "custom_resource_definition.go",
         "etcd_failure.go",
@@ -36,9 +37,11 @@ go_library(
         "//staging/src/k8s.io/api/batch/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/test/integration:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",

test/e2e/apimachinery/webhook.go

@@ -136,7 +136,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
 		defer testcrd.CleanUp()
 		webhookCleanup := registerWebhookForCustomResource(f, context, testcrd)
 		defer webhookCleanup()
-		testCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClient)
+		testCustomResourceWebhook(f, testcrd.Crd, testcrd.GetV1DynamicClient())
 	})
 
 	It("Should unconditionally reject operations on fail closed webhook", func() {
@@ -173,7 +173,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
 		defer testcrd.CleanUp()
 		webhookCleanup := registerMutatingWebhookForCustomResource(f, context, testcrd)
 		defer webhookCleanup()
-		testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClient)
+		testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.GetV1DynamicClient())
 	})
 
 	It("Should deny crd creation", func() {
@@ -1157,7 +1157,7 @@ func registerWebhookForCustomResource(f *framework.Framework, context *certConte
 					Operations: []v1beta1.OperationType{v1beta1.Create},
 					Rule: v1beta1.Rule{
 						APIGroups:   []string{testcrd.ApiGroup},
-						APIVersions: []string{testcrd.ApiVersion},
+						APIVersions: testcrd.GetAPIVersions(),
 						Resources:   []string{testcrd.GetPluralName()},
 					},
 				}},
@@ -1198,7 +1198,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c
 					Operations: []v1beta1.OperationType{v1beta1.Create},
 					Rule: v1beta1.Rule{
 						APIGroups:   []string{testcrd.ApiGroup},
-						APIVersions: []string{testcrd.ApiVersion},
+						APIVersions: testcrd.GetAPIVersions(),
 						Resources:   []string{testcrd.GetPluralName()},
 					},
 				}},
@@ -1217,7 +1217,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c
 					Operations: []v1beta1.OperationType{v1beta1.Create},
 					Rule: v1beta1.Rule{
 						APIGroups:   []string{testcrd.ApiGroup},
-						APIVersions: []string{testcrd.ApiVersion},
+						APIVersions: testcrd.GetAPIVersions(),
 						Resources:   []string{testcrd.GetPluralName()},
 					},
 				}},
@@ -1343,12 +1343,18 @@ func testCRDDenyWebhook(f *framework.Framework) {
 	name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, "deny")
 	kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, "deny")
 	group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
-	apiVersion := "v1"
+	apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+		{
+			Name:    "v1",
+			Served:  true,
+			Storage: true,
+		},
+	}
 	testcrd := &framework.TestCrd{
-		Name:       name,
-		Kind:       kind,
-		ApiGroup:   group,
-		ApiVersion: apiVersion,
+		Name:     name,
+		Kind:     kind,
+		ApiGroup: group,
+		Versions: apiVersions,
 	}
 
 	// Creating a custom resource definition for use by assorted tests.
@@ -1370,8 +1376,8 @@ func testCRDDenyWebhook(f *framework.Framework) {
 			},
 		},
 		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:   testcrd.ApiGroup,
-			Version: testcrd.ApiVersion,
+			Group:    testcrd.ApiGroup,
+			Versions: testcrd.Versions,
 			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
 				Plural:   testcrd.GetPluralName(),
 				Singular: testcrd.Name,

test/e2e/framework/crd_util.go

@@ -35,25 +35,23 @@ type TestCrd struct {
 	Name               string
 	Kind               string
 	ApiGroup           string
-	ApiVersion         string
+	Versions           []apiextensionsv1beta1.CustomResourceDefinitionVersion
 	ApiExtensionClient *crdclientset.Clientset
 	Crd                *apiextensionsv1beta1.CustomResourceDefinition
-	DynamicClient      dynamic.ResourceInterface
+	DynamicClients     map[string]dynamic.ResourceInterface
 	CleanUp            CleanCrdFn
 }
 
 // CreateTestCRD creates a new CRD specifically for the calling test.
-func CreateTestCRD(f *Framework) (*TestCrd, error) {
+func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiextensionsv1beta1.CustomResourceDefinitionVersion, conversionWebhook *apiextensionsv1beta1.WebhookClientConfig) (*TestCrd, error) {
 	suffix := randomSuffix()
 	name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, suffix)
 	kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, suffix)
-	group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
-	apiVersion := "v1"
 	testcrd := &TestCrd{
-		Name:       name,
-		Kind:       kind,
-		ApiGroup:   group,
-		ApiVersion: apiVersion,
+		Name:     name,
+		Kind:     kind,
+		ApiGroup: group,
+		Versions: apiVersions,
 	}
 
 	// Creating a custom resource definition for use by assorted tests.
@@ -75,6 +73,13 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) {
 
 	crd := newCRDForTest(testcrd)
 
+	if conversionWebhook != nil {
+		crd.Spec.Conversion = &apiextensionsv1beta1.CustomResourceConversion{
+			Strategy:            "Webhook",
+			WebhookClientConfig: conversionWebhook,
+		}
+	}
+
 	//create CRD and waits for the resource to be recognized and available.
 	crd, err = fixtures.CreateNewCustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient)
 	if err != nil {
@@ -82,12 +87,17 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) {
 		return nil, err
 	}
 
-	gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: crd.Spec.Version, Resource: crd.Spec.Names.Plural}
-	resourceClient := dynamicClient.Resource(gvr).Namespace(f.Namespace.Name)
+	resourceClients := map[string]dynamic.ResourceInterface{}
+	for _, v := range crd.Spec.Versions {
+		if v.Served {
+			gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: v.Name, Resource: crd.Spec.Names.Plural}
+			resourceClients[v.Name] = dynamicClient.Resource(gvr).Namespace(f.Namespace.Name)
+		}
+	}
 
 	testcrd.ApiExtensionClient = apiExtensionClient
 	testcrd.Crd = crd
-	testcrd.DynamicClient = resourceClient
+	testcrd.DynamicClients = resourceClients
 	testcrd.CleanUp = func() error {
 		err := fixtures.DeleteCustomResourceDefinition(crd, apiExtensionClient)
 		if err != nil {
@@ -98,13 +108,26 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) {
 	return testcrd, nil
 }
 
+// CreateTestCRD creates a new CRD specifically for the calling test.
+func CreateTestCRD(f *Framework) (*TestCrd, error) {
+	group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
+	apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+		{
+			Name:    "v1",
+			Served:  true,
+			Storage: true,
+		},
+	}
+	return CreateMultiVersionTestCRD(f, group, apiVersions, nil)
+}
+
 // newCRDForTest generates a CRD definition for the test
 func newCRDForTest(testcrd *TestCrd) *apiextensionsv1beta1.CustomResourceDefinition {
 	return &apiextensionsv1beta1.CustomResourceDefinition{
 		ObjectMeta: metav1.ObjectMeta{Name: testcrd.GetMetaName()},
 		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:   testcrd.ApiGroup,
-			Version: testcrd.ApiVersion,
+			Group:    testcrd.ApiGroup,
+			Versions: testcrd.Versions,
 			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
 				Plural:   testcrd.GetPluralName(),
 				Singular: testcrd.Name,
@@ -130,3 +153,17 @@ func (c *TestCrd) GetPluralName() string {
 func (c *TestCrd) GetListName() string {
 	return c.Name + "List"
 }
+
+func (c *TestCrd) GetAPIVersions() []string {
+	ret := []string{}
+	for _, v := range c.Versions {
+		if v.Served {
+			ret = append(ret, v.Name)
+		}
+	}
+	return ret
+}
+
+func (c *TestCrd) GetV1DynamicClient() dynamic.ResourceInterface {
+	return c.DynamicClients["v1"]
+}

test/images/BUILD

@@ -12,6 +12,7 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//test/images/apparmor-loader:all-srcs",
+        "//test/images/crd-conversion-webhook:all-srcs",
         "//test/images/echoserver:all-srcs",
         "//test/images/entrypoint-tester:all-srcs",
         "//test/images/fakegitserver:all-srcs",

test/images/crd-conversion-webhook/BASEIMAGE

+amd64=alpine:3.6
+arm=arm32v6/alpine:3.6
+arm64=arm64v8/alpine:3.6
+ppc64le=ppc64le/alpine:3.6

test/images/crd-conversion-webhook/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "main.go",
+    ],
+    importpath = "k8s.io/kubernetes/test/images/crd-conversion-webhook",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
+        "//test/images/crd-conversion-webhook/converter:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "crd-conversion-webhook",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//test/images/crd-conversion-webhook/converter:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/images/crd-conversion-webhook/Dockerfile

+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM BASEIMAGE
+
+ADD crd_conversion_webhook /crd_conversion_webhook
+ENTRYPOINT ["/crd_conversion_webhook"]

test/images/crd-conversion-webhook/Makefile

+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SRCS=crd_conversion_webhook
+ARCH ?= amd64
+TARGET ?= $(CURDIR)
+GOLANG_VERSION ?= latest
+SRC_DIR = $(notdir $(shell pwd))
+export
+
+bin:
+	../image-util.sh bin $(SRCS)
+
+.PHONY: bin

test/images/crd-conversion-webhook/README.md

+# Kubernetes External Admission Webhook Test Image
+
+The image tests CustomResourceConversionWebhook. After deploying it to kubernetes cluster,
+administrator needs to create a CustomResourceConversion.Webhook
+in kubernetes cluster to use remote webhook for conversions.
+
+## Build the code
+
+```bash
+make build
+```

test/images/crd-conversion-webhook/VERSION

+1.13rev2

test/images/crd-conversion-webhook/config.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"crypto/tls"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+
+	"github.com/golang/glog"
+)
+
+// Get a clientset with in-cluster config.
+func getClient() *kubernetes.Clientset {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		glog.Fatal(err)
+	}
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		glog.Fatal(err)
+	}
+	return clientset
+}
+
+func configTLS(config Config, clientset *kubernetes.Clientset) *tls.Config {
+	sCert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile)
+	if err != nil {
+		glog.Fatal(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{sCert},
+		// TODO: uses mutual tls after we agree on what cert the apiserver should use.
+		// ClientAuth:   tls.RequireAndVerifyClientCert,
+	}
+}

test/images/crd-conversion-webhook/converter/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "example_converter.go",
+        "framework.go",
+    ],
+    importpath = "k8s.io/kubernetes/test/images/crd-conversion-webhook/converter",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library",
+        "//vendor/bitbucket.org/ww/goautoneg:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["converter_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/images/crd-conversion-webhook/converter/converter_test.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package converter
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer/json"
+)
+
+func TestConverter(t *testing.T) {
+	sampleObj := `kind: ConversionReview
+apiVersion: apiextensions.k8s.io/v1beta1
+request:
+  uid: 0000-0000-0000-0000
+  desiredAPIVersion: stable.example.com/v2
+  objects:
+    - apiVersion: stable.example.com/v1
+      kind: CronTab
+      metadata:
+        name: my-new-cron-object
+      spec:
+        cronSpec: "* * * * */5"
+        image: my-awesome-cron-image
+      hostPort: "localhost:7070"
+`
+	// First try json, it should fail as the data is taml
+	response := httptest.NewRecorder()
+	request, err := http.NewRequest("POST", "/convert", strings.NewReader(sampleObj))
+	if err != nil {
+		t.Fatal(err)
+	}
+	request.Header.Add("Content-Type", "application/json")
+	ServeExampleConvert(response, request)
+	convertReview := v1beta1.ConversionReview{}
+	scheme := runtime.NewScheme()
+	jsonSerializer := json.NewSerializer(json.DefaultMetaFactory, scheme, scheme, false)
+	if _, _, err := jsonSerializer.Decode(response.Body.Bytes(), nil, &convertReview); err != nil {
+		t.Fatal(err)
+	}
+	if convertReview.Response.Result.Status != v1.StatusFailure {
+		t.Fatalf("expected the operation to fail when yaml is provided with json header")
+	} else if !strings.Contains(convertReview.Response.Result.Message, "json parse error") {
+		t.Fatalf("expected to fail on json parser, but it failed with: %v", convertReview.Response.Result.Message)
+	}
+
+	// Now try yaml, and it should successfully convert
+	response = httptest.NewRecorder()
+	request, err = http.NewRequest("POST", "/convert", strings.NewReader(sampleObj))
+	if err != nil {
+		t.Fatal(err)
+	}
+	request.Header.Add("Content-Type", "application/yaml")
+	ServeExampleConvert(response, request)
+	convertReview = v1beta1.ConversionReview{}
+	yamlSerializer := json.NewYAMLSerializer(json.DefaultMetaFactory, scheme, scheme)
+	if _, _, err := yamlSerializer.Decode(response.Body.Bytes(), nil, &convertReview); err != nil {
+		t.Fatalf("cannot decode data: \n %v\n Error: %v", response.Body, err)
+	}
+	if convertReview.Response.Result.Status != v1.StatusSuccess {
+		t.Fatalf("cr conversion failed: %v", convertReview.Response)
+	}
+	convertedObj := unstructured.Unstructured{}
+	if _, _, err := yamlSerializer.Decode(convertReview.Response.ConvertedObjects[0].Raw, nil, &convertedObj); err != nil {
+		t.Fatal(err)
+	}
+	if e, a := "stable.example.com/v2", convertedObj.GetAPIVersion(); e != a {
+		t.Errorf("expected= %v, actual= %v", e, a)
+	}
+	if e, a := "localhost", convertedObj.Object["host"]; e != a {
+		t.Errorf("expected= %v, actual= %v", e, a)
+	}
+	if e, a := "7070", convertedObj.Object["port"]; e != a {
+		t.Errorf("expected= %v, actual= %v", e, a)
+	}
+}

test/images/crd-conversion-webhook/converter/example_converter.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package converter
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/golang/glog"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+func convertExampleCRD(Object *unstructured.Unstructured, toVersion string) (*unstructured.Unstructured, metav1.Status) {
+	glog.V(2).Info("converting crd")
+
+	convertedObject := Object.DeepCopy()
+	fromVersion := Object.GetAPIVersion()
+
+	if toVersion == fromVersion {
+		return nil, statusErrorWithMessage("conversion from a version to itself should not call the webhook: %s", toVersion)
+	}
+
+	switch Object.GetAPIVersion() {
+	case "stable.example.com/v1":
+		switch toVersion {
+		case "stable.example.com/v2":
+			hostPort, ok := convertedObject.Object["hostPort"]
+			if ok {
+				delete(convertedObject.Object, "hostPort")
+				parts := strings.Split(hostPort.(string), ":")
+				if len(parts) != 2 {
+					return nil, statusErrorWithMessage("invalid hostPort value `%v`", hostPort)
+				}
+				convertedObject.Object["host"] = parts[0]
+				convertedObject.Object["port"] = parts[1]
+			}
+		default:
+			return nil, statusErrorWithMessage("unexpected conversion version %q", toVersion)
+		}
+	case "stable.example.com/v2":
+		switch toVersion {
+		case "stable.example.com/v1":
+			host, hasHost := convertedObject.Object["host"]
+			port, hasPort := convertedObject.Object["port"]
+			if hasHost || hasPort {
+				if !hasHost {
+					host = ""
+				}
+				if !hasPort {
+					port = ""
+				}
+				convertedObject.Object["hostPort"] = fmt.Sprintf("%s:%s", host, port)
+				delete(convertedObject.Object, "host")
+				delete(convertedObject.Object, "port")
+			}
+		default:
+			return nil, statusErrorWithMessage("unexpected conversion version %q", toVersion)
+		}
+	default:
+		return nil, statusErrorWithMessage("unexpected conversion version %q", fromVersion)
+	}
+	return convertedObject, statusSucceed()
+}

test/images/crd-conversion-webhook/converter/framework.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package converter
+
+import (
+	"bitbucket.org/ww/goautoneg"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/golang/glog"
+
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer/json"
+)
+
+// convertFunc is the user defined function for any conversion. The code in this file is a
+// template that can be use for any CR conversion given this function.
+type convertFunc func(Object *unstructured.Unstructured, version string) (*unstructured.Unstructured, metav1.Status)
+
+// conversionResponseFailureWithMessagef is a helper function to create an AdmissionResponse
+// with a formatted embedded error message.
+func conversionResponseFailureWithMessagef(msg string, params ...interface{}) *v1beta1.ConversionResponse {
+	return &v1beta1.ConversionResponse{
+		Result: metav1.Status{
+			Message: fmt.Sprintf(msg, params...),
+			Status:  metav1.StatusFailure,
+		},
+	}
+
+}
+
+func statusErrorWithMessage(msg string, params ...interface{}) metav1.Status {
+	return metav1.Status{
+		Message: fmt.Sprintf(msg, params...),
+		Status:  metav1.StatusFailure,
+	}
+}
+
+func statusSucceed() metav1.Status {
+	return metav1.Status{
+		Status: metav1.StatusSuccess,
+	}
+}
+
+// doConversion converts the requested object given the conversion function and returns a conversion response.
+// failures will be reported as Reason in the conversion response.
+func doConversion(convertRequest *v1beta1.ConversionRequest, convert convertFunc) *v1beta1.ConversionResponse {
+	var convertedObjects []runtime.RawExtension
+	for _, obj := range convertRequest.Objects {
+		cr := unstructured.Unstructured{}
+		if err := cr.UnmarshalJSON(obj.Raw); err != nil {
+			glog.Error(err)
+			return conversionResponseFailureWithMessagef("failed to unmarshall object (%v) with error: %v", string(obj.Raw), err)
+		}
+		convertedCR, status := convert(&cr, convertRequest.DesiredAPIVersion)
+		if status.Status != metav1.StatusSuccess {
+			glog.Error(status.String())
+			return &v1beta1.ConversionResponse{
+				Result: status,
+			}
+		}
+		convertedCR.SetAPIVersion(convertRequest.DesiredAPIVersion)
+		convertedObjects = append(convertedObjects, runtime.RawExtension{Object: convertedCR})
+	}
+	return &v1beta1.ConversionResponse{
+		ConvertedObjects: convertedObjects,
+		Result:           statusSucceed(),
+	}
+}
+
+func serve(w http.ResponseWriter, r *http.Request, convert convertFunc) {
+	var body []byte
+	if r.Body != nil {
+		if data, err := ioutil.ReadAll(r.Body); err == nil {
+			body = data
+		}
+	}
+
+	contentType := r.Header.Get("Content-Type")
+	serializer := getInputSerializer(contentType)
+	if serializer == nil {
+		msg := fmt.Sprintf("invalid Content-Type header `%s`", contentType)
+		glog.Errorf(msg)
+		http.Error(w, msg, http.StatusBadRequest)
+		return
+	}
+
+	glog.V(2).Infof("handling request: %v", body)
+	convertReview := v1beta1.ConversionReview{}
+	if _, _, err := serializer.Decode(body, nil, &convertReview); err != nil {
+		glog.Error(err)
+		convertReview.Response = conversionResponseFailureWithMessagef("failed to deserialize body (%v) with error %v", string(body), err)
+	} else {
+		convertReview.Response = doConversion(convertReview.Request, convert)
+		convertReview.Response.UID = convertReview.Request.UID
+	}
+	glog.V(2).Info(fmt.Sprintf("sending response: %v", convertReview.Response))
+
+	// reset the request, it is not needed in a response.
+	convertReview.Request = &v1beta1.ConversionRequest{}
+
+	accept := r.Header.Get("Accept")
+	outSerializer := getOutputSerializer(accept)
+	if outSerializer == nil {
+		msg := fmt.Sprintf("invalid accept header `%s`", accept)
+		glog.Errorf(msg)
+		http.Error(w, msg, http.StatusBadRequest)
+		return
+	}
+	err := outSerializer.Encode(&convertReview, w)
+	if err != nil {
+		glog.Error(err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+// ServeExampleConvert servers endpoint for the example converter defined as convertExampleCRD function.
+func ServeExampleConvert(w http.ResponseWriter, r *http.Request) {
+	serve(w, r, convertExampleCRD)
+}
+
+type mediaType struct {
+	Type, SubType string
+}
+
+var scheme = runtime.NewScheme()
+var serializers = map[mediaType]runtime.Serializer{
+	{"application", "json"}: json.NewSerializer(json.DefaultMetaFactory, scheme, scheme, false),
+	{"application", "yaml"}: json.NewYAMLSerializer(json.DefaultMetaFactory, scheme, scheme),
+}
+
+func getInputSerializer(contentType string) runtime.Serializer {
+	parts := strings.SplitN(contentType, "/", 2)
+	if len(parts) != 2 {
+		return nil
+	}
+	return serializers[mediaType{parts[0], parts[1]}]
+}
+
+func getOutputSerializer(accept string) runtime.Serializer {
+	if len(accept) == 0 {
+		return serializers[mediaType{"application", "json"}]
+	}
+
+	clauses := goautoneg.ParseAccept(accept)
+	for _, clause := range clauses {
+		for k, v := range serializers {
+			switch {
+			case clause.Type == k.Type && clause.SubType == k.SubType,
+				clause.Type == k.Type && clause.SubType == "*",
+				clause.Type == "*" && clause.SubType == "*":
+				return v
+			}
+		}
+	}
+
+	return nil
+}

test/images/crd-conversion-webhook/main.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"net/http"
+
+	"k8s.io/kubernetes/test/images/crd-conversion-webhook/converter"
+)
+
+// Config contains the server (the webhook) cert and key.
+type Config struct {
+	CertFile string
+	KeyFile  string
+}
+
+func (c *Config) addFlags() {
+	flag.StringVar(&c.CertFile, "tls-cert-file", c.CertFile, ""+
+		"File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
+		"after server cert).")
+	flag.StringVar(&c.KeyFile, "tls-private-key-file", c.KeyFile, ""+
+		"File containing the default x509 private key matching --tls-cert-file.")
+}
+
+func main() {
+	var config Config
+	config.addFlags()
+	flag.Parse()
+
+	http.HandleFunc("/crdconvert", converter.ServeExampleConvert)
+	clientset := getClient()
+	server := &http.Server{
+		Addr:      ":443",
+		TLSConfig: configTLS(config, clientset),
+	}
+	server.ListenAndServeTLS("", "")
+}

test/utils/image/manifest.go

@@ -92,6 +92,7 @@ var (
 
 // Preconfigured image configs
 var (
+	CRDConversionWebhook     = Config{e2eRegistry, "crd-conversion-webhook", "1.13rev2"}
 	AdmissionWebhook         = Config{e2eRegistry, "webhook", "1.13v1"}
 	APIServer                = Config{e2eRegistry, "sample-apiserver", "1.10"}
 	AppArmorLoader           = Config{e2eRegistry, "apparmor-loader", "1.0"}