Merge pull request #70211 from roycaihw/crd-multi-version-x

api/openapi-spec/swagger.json

@@ -91983,7 +91983,7 @@
     ],
     "properties": {
      "additionalPrinterColumns": {
-      "description": "AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column.",
+      "description": "AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column. Optional, the global columns for all versions. Top-level and per-version columns are mutually exclusive.",
       "type": "array",
       "items": {
        "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceColumnDefinition"
@@ -92006,11 +92006,11 @@
       "type": "string"
      },
      "subresources": {
-      "description": "Subresources describes the subresources for CustomResources",
+      "description": "Subresources describes the subresources for CustomResource Optional, the global subresources for all versions. Top-level and per-version subresources are mutually exclusive.",
       "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceSubresources"
      },
      "validation": {
-      "description": "Validation describes the validation methods for CustomResources",
+      "description": "Validation describes the validation methods for CustomResources Optional, the global validation schema for all versions. Top-level and per-version schemas are mutually exclusive.",
       "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceValidation"
      },
      "version": {
@@ -92062,10 +92062,21 @@
      "storage"
     ],
     "properties": {
+     "additionalPrinterColumns": {
+      "description": "AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column. Top-level and per-version columns are mutually exclusive. Per-version columns must not all be set to identical values (top-level columns should be used instead) This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature. NOTE: CRDs created prior to 1.13 populated the top-level additionalPrinterColumns field by default. To apply an update that changes to per-version additionalPrinterColumns, the top-level additionalPrinterColumns field must be explicitly set to null",
+      "type": "array",
+      "items": {
+       "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceColumnDefinition"
+      }
+     },
      "name": {
       "description": "Name is the version name, e.g. “v1”, “v2beta1”, etc.",
       "type": "string"
      },
+     "schema": {
+      "description": "Schema describes the schema for CustomResource used in validation, pruning, and defaulting. Top-level and per-version schemas are mutually exclusive. Per-version schemas must not all be set to identical values (top-level validation schema should be used instead) This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.",
+      "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceValidation"
+     },
      "served": {
       "description": "Served is a flag enabling/disabling this version from being served via REST APIs",
       "type": "boolean"
@@ -92073,6 +92084,10 @@
      "storage": {
       "description": "Storage flags the version as storage version. There must be exactly one flagged as storage version.",
       "type": "boolean"
+     },
+     "subresources": {
+      "description": "Subresources describes the subresources for CustomResource Top-level and per-version subresources are mutually exclusive. Per-version subresources must not all be set to identical values (top-level subresources should be used instead) This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.",
+      "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceSubresources"
      }
     }
    },

staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json

@@ -2443,6 +2443,10 @@
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
 		{
+			"ImportPath": "k8s.io/apiserver/pkg/util/feature/testing",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
 			"ImportPath": "k8s.io/apiserver/pkg/util/logs",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go

@@ -44,8 +44,14 @@ type CustomResourceDefinitionSpec struct {
 	// Scope indicates whether this resource is cluster or namespace scoped.  Default is namespaced
 	Scope ResourceScope
 	// Validation describes the validation methods for CustomResources
+	// Optional, the global validation schema for all versions.
+	// Top-level and per-version schemas are mutually exclusive.
+	// +optional
 	Validation *CustomResourceValidation
-	// Subresources describes the subresources for CustomResources
+	// Subresources describes the subresources for CustomResource
+	// Optional, the global subresources for all versions.
+	// Top-level and per-version subresources are mutually exclusive.
+	// +optional
 	Subresources *CustomResourceSubresources
 	// Versions is the list of all supported versions for this resource.
 	// If Version field is provided, this field is optional.
@@ -60,6 +66,9 @@ type CustomResourceDefinitionSpec struct {
 	// v10, v2, v1, v11beta2, v10beta3, v3beta1, v12alpha1, v11alpha2, foo1, foo10.
 	Versions []CustomResourceDefinitionVersion
 	// AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column.
+	// Optional, the global columns for all versions.
+	// Top-level and per-version columns are mutually exclusive.
+	// +optional
 	AdditionalPrinterColumns []CustomResourceColumnDefinition
 
 	// `conversion` defines conversion settings for the CRD.
@@ -149,6 +158,27 @@ type CustomResourceDefinitionVersion struct {
 	// Storage flags the version as storage version. There must be exactly one flagged
 	// as storage version.
 	Storage bool
+	// Schema describes the schema for CustomResource used in validation, pruning, and defaulting.
+	// Top-level and per-version schemas are mutually exclusive.
+	// Per-version schemas must not all be set to identical values (top-level validation schema should be used instead)
+	// This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.
+	// +optional
+	Schema *CustomResourceValidation
+	// Subresources describes the subresources for CustomResource
+	// Top-level and per-version subresources are mutually exclusive.
+	// Per-version subresources must not all be set to identical values (top-level subresources should be used instead)
+	// This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.
+	// +optional
+	Subresources *CustomResourceSubresources
+	// AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column.
+	// Top-level and per-version columns are mutually exclusive.
+	// Per-version columns must not all be set to identical values (top-level columns should be used instead)
+	// This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.
+	// NOTE: CRDs created prior to 1.13 populated the top-level additionalPrinterColumns field by default. To apply an
+	// update that changes to per-version additionalPrinterColumns, the top-level additionalPrinterColumns field must
+	// be explicitly set to null
+	// +optional
+	AdditionalPrinterColumns []CustomResourceColumnDefinition
 }
 
 // CustomResourceColumnDefinition specifies a column for server side printing.

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/defaults.go

@@ -19,12 +19,9 @@ package v1beta1
 import (
 	"strings"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
-var swaggerMetadataDescriptions = metav1.ObjectMeta{}.SwaggerDoc()
-
 func addDefaultingFuncs(scheme *runtime.Scheme) error {
 	scheme.AddTypeDefaultingFunc(&CustomResourceDefinition{}, func(obj interface{}) { SetDefaults_CustomResourceDefinition(obj.(*CustomResourceDefinition)) })
 	// TODO figure out why I can't seem to get my defaulter generated
@@ -66,14 +63,19 @@ func SetDefaults_CustomResourceDefinitionSpec(obj *CustomResourceDefinitionSpec)
 	if len(obj.Version) == 0 && len(obj.Versions) != 0 {
 		obj.Version = obj.Versions[0].Name
 	}
-	if len(obj.AdditionalPrinterColumns) == 0 {
-		obj.AdditionalPrinterColumns = []CustomResourceColumnDefinition{
-			{Name: "Age", Type: "date", Description: swaggerMetadataDescriptions["creationTimestamp"], JSONPath: ".metadata.creationTimestamp"},
-		}
-	}
 	if obj.Conversion == nil {
 		obj.Conversion = &CustomResourceConversion{
 			Strategy: NoneConverter,
 		}
 	}
 }
+
+// hasPerVersionColumns returns true if a CRD uses per-version columns.
+func hasPerVersionColumns(versions []CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if len(v.AdditionalPrinterColumns) > 0 {
+			return true
+		}
+	}
+	return false
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto

@@ -203,10 +203,14 @@ message CustomResourceDefinitionSpec {
   optional string scope = 4;
 
   // Validation describes the validation methods for CustomResources
+  // Optional, the global validation schema for all versions.
+  // Top-level and per-version schemas are mutually exclusive.
   // +optional
   optional CustomResourceValidation validation = 5;
 
-  // Subresources describes the subresources for CustomResources
+  // Subresources describes the subresources for CustomResource
+  // Optional, the global subresources for all versions.
+  // Top-level and per-version subresources are mutually exclusive.
   // +optional
   optional CustomResourceSubresources subresources = 6;
 
@@ -225,6 +229,8 @@ message CustomResourceDefinitionSpec {
   repeated CustomResourceDefinitionVersion versions = 7;
 
   // AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column.
+  // Optional, the global columns for all versions.
+  // Top-level and per-version columns are mutually exclusive.
   // +optional
   repeated CustomResourceColumnDefinition additionalPrinterColumns = 8;
 
@@ -262,6 +268,30 @@ message CustomResourceDefinitionVersion {
   // Storage flags the version as storage version. There must be exactly one
   // flagged as storage version.
   optional bool storage = 3;
+
+  // Schema describes the schema for CustomResource used in validation, pruning, and defaulting.
+  // Top-level and per-version schemas are mutually exclusive.
+  // Per-version schemas must not all be set to identical values (top-level validation schema should be used instead)
+  // This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.
+  // +optional
+  optional CustomResourceValidation schema = 4;
+
+  // Subresources describes the subresources for CustomResource
+  // Top-level and per-version subresources are mutually exclusive.
+  // Per-version subresources must not all be set to identical values (top-level subresources should be used instead)
+  // This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.
+  // +optional
+  optional CustomResourceSubresources subresources = 5;
+
+  // AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column.
+  // Top-level and per-version columns are mutually exclusive.
+  // Per-version columns must not all be set to identical values (top-level columns should be used instead)
+  // This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.
+  // NOTE: CRDs created prior to 1.13 populated the top-level additionalPrinterColumns field by default. To apply an
+  // update that changes to per-version additionalPrinterColumns, the top-level additionalPrinterColumns field must
+  // be explicitly set to null
+  // +optional
+  repeated CustomResourceColumnDefinition additionalPrinterColumns = 6;
 }
 
 // CustomResourceSubresourceScale defines how to serve the scale subresource for CustomResources.

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go

@@ -47,9 +47,13 @@ type CustomResourceDefinitionSpec struct {
 	// Scope indicates whether this resource is cluster or namespace scoped.  Default is namespaced
 	Scope ResourceScope `json:"scope" protobuf:"bytes,4,opt,name=scope,casttype=ResourceScope"`
 	// Validation describes the validation methods for CustomResources
+	// Optional, the global validation schema for all versions.
+	// Top-level and per-version schemas are mutually exclusive.
 	// +optional
 	Validation *CustomResourceValidation `json:"validation,omitempty" protobuf:"bytes,5,opt,name=validation"`
-	// Subresources describes the subresources for CustomResources
+	// Subresources describes the subresources for CustomResource
+	// Optional, the global subresources for all versions.
+	// Top-level and per-version subresources are mutually exclusive.
 	// +optional
 	Subresources *CustomResourceSubresources `json:"subresources,omitempty" protobuf:"bytes,6,opt,name=subresources"`
 	// Versions is the list of all supported versions for this resource.
@@ -66,6 +70,8 @@ type CustomResourceDefinitionSpec struct {
 	// +optional
 	Versions []CustomResourceDefinitionVersion `json:"versions,omitempty" protobuf:"bytes,7,rep,name=versions"`
 	// AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column.
+	// Optional, the global columns for all versions.
+	// Top-level and per-version columns are mutually exclusive.
 	// +optional
 	AdditionalPrinterColumns []CustomResourceColumnDefinition `json:"additionalPrinterColumns,omitempty" protobuf:"bytes,8,rep,name=additionalPrinterColumns"`
 
@@ -159,6 +165,27 @@ type CustomResourceDefinitionVersion struct {
 	// Storage flags the version as storage version. There must be exactly one
 	// flagged as storage version.
 	Storage bool `json:"storage" protobuf:"varint,3,opt,name=storage"`
+	// Schema describes the schema for CustomResource used in validation, pruning, and defaulting.
+	// Top-level and per-version schemas are mutually exclusive.
+	// Per-version schemas must not all be set to identical values (top-level validation schema should be used instead)
+	// This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.
+	// +optional
+	Schema *CustomResourceValidation `json:"schema,omitempty" protobuf:"bytes,4,opt,name=schema"`
+	// Subresources describes the subresources for CustomResource
+	// Top-level and per-version subresources are mutually exclusive.
+	// Per-version subresources must not all be set to identical values (top-level subresources should be used instead)
+	// This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.
+	// +optional
+	Subresources *CustomResourceSubresources `json:"subresources,omitempty" protobuf:"bytes,5,opt,name=subresources"`
+	// AdditionalPrinterColumns are additional columns shown e.g. in kubectl next to the name. Defaults to a created-at column.
+	// Top-level and per-version columns are mutually exclusive.
+	// Per-version columns must not all be set to identical values (top-level columns should be used instead)
+	// This field is alpha-level and is only honored by servers that enable the CustomResourceWebhookConversion feature.
+	// NOTE: CRDs created prior to 1.13 populated the top-level additionalPrinterColumns field by default. To apply an
+	// update that changes to per-version additionalPrinterColumns, the top-level additionalPrinterColumns field must
+	// be explicitly set to null
+	// +optional
+	AdditionalPrinterColumns []CustomResourceColumnDefinition `json:"additionalPrinterColumns,omitempty" protobuf:"bytes,6,rep,name=additionalPrinterColumns"`
 }
 
 // CustomResourceColumnDefinition specifies a column for server side printing.

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go

@@ -464,7 +464,17 @@ func autoConvert_v1beta1_CustomResourceDefinitionSpec_To_apiextensions_CustomRes
 		out.Validation = nil
 	}
 	out.Subresources = (*apiextensions.CustomResourceSubresources)(unsafe.Pointer(in.Subresources))
-	out.Versions = *(*[]apiextensions.CustomResourceDefinitionVersion)(unsafe.Pointer(&in.Versions))
+	if in.Versions != nil {
+		in, out := &in.Versions, &out.Versions
+		*out = make([]apiextensions.CustomResourceDefinitionVersion, len(*in))
+		for i := range *in {
+			if err := Convert_v1beta1_CustomResourceDefinitionVersion_To_apiextensions_CustomResourceDefinitionVersion(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.Versions = nil
+	}
 	out.AdditionalPrinterColumns = *(*[]apiextensions.CustomResourceColumnDefinition)(unsafe.Pointer(&in.AdditionalPrinterColumns))
 	out.Conversion = (*apiextensions.CustomResourceConversion)(unsafe.Pointer(in.Conversion))
 	return nil
@@ -492,7 +502,17 @@ func autoConvert_apiextensions_CustomResourceDefinitionSpec_To_v1beta1_CustomRes
 		out.Validation = nil
 	}
 	out.Subresources = (*CustomResourceSubresources)(unsafe.Pointer(in.Subresources))
-	out.Versions = *(*[]CustomResourceDefinitionVersion)(unsafe.Pointer(&in.Versions))
+	if in.Versions != nil {
+		in, out := &in.Versions, &out.Versions
+		*out = make([]CustomResourceDefinitionVersion, len(*in))
+		for i := range *in {
+			if err := Convert_apiextensions_CustomResourceDefinitionVersion_To_v1beta1_CustomResourceDefinitionVersion(&(*in)[i], &(*out)[i], s); err != nil {
+				return err
+			}
+		}
+	} else {
+		out.Versions = nil
+	}
 	out.AdditionalPrinterColumns = *(*[]CustomResourceColumnDefinition)(unsafe.Pointer(&in.AdditionalPrinterColumns))
 	out.Conversion = (*CustomResourceConversion)(unsafe.Pointer(in.Conversion))
 	return nil
@@ -535,6 +555,17 @@ func autoConvert_v1beta1_CustomResourceDefinitionVersion_To_apiextensions_Custom
 	out.Name = in.Name
 	out.Served = in.Served
 	out.Storage = in.Storage
+	if in.Schema != nil {
+		in, out := &in.Schema, &out.Schema
+		*out = new(apiextensions.CustomResourceValidation)
+		if err := Convert_v1beta1_CustomResourceValidation_To_apiextensions_CustomResourceValidation(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Schema = nil
+	}
+	out.Subresources = (*apiextensions.CustomResourceSubresources)(unsafe.Pointer(in.Subresources))
+	out.AdditionalPrinterColumns = *(*[]apiextensions.CustomResourceColumnDefinition)(unsafe.Pointer(&in.AdditionalPrinterColumns))
 	return nil
 }
 
@@ -547,6 +578,17 @@ func autoConvert_apiextensions_CustomResourceDefinitionVersion_To_v1beta1_Custom
 	out.Name = in.Name
 	out.Served = in.Served
 	out.Storage = in.Storage
+	if in.Schema != nil {
+		in, out := &in.Schema, &out.Schema
+		*out = new(CustomResourceValidation)
+		if err := Convert_apiextensions_CustomResourceValidation_To_v1beta1_CustomResourceValidation(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Schema = nil
+	}
+	out.Subresources = (*CustomResourceSubresources)(unsafe.Pointer(in.Subresources))
+	out.AdditionalPrinterColumns = *(*[]CustomResourceColumnDefinition)(unsafe.Pointer(&in.AdditionalPrinterColumns))
 	return nil
 }
 

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.deepcopy.go

@@ -264,7 +264,9 @@ func (in *CustomResourceDefinitionSpec) DeepCopyInto(out *CustomResourceDefiniti
 	if in.Versions != nil {
 		in, out := &in.Versions, &out.Versions
 		*out = make([]CustomResourceDefinitionVersion, len(*in))
-		copy(*out, *in)
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
 	}
 	if in.AdditionalPrinterColumns != nil {
 		in, out := &in.AdditionalPrinterColumns, &out.AdditionalPrinterColumns
@@ -321,6 +323,21 @@ func (in *CustomResourceDefinitionStatus) DeepCopy() *CustomResourceDefinitionSt
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CustomResourceDefinitionVersion) DeepCopyInto(out *CustomResourceDefinitionVersion) {
 	*out = *in
+	if in.Schema != nil {
+		in, out := &in.Schema, &out.Schema
+		*out = new(CustomResourceValidation)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Subresources != nil {
+		in, out := &in.Subresources, &out.Subresources
+		*out = new(CustomResourceSubresources)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.AdditionalPrinterColumns != nil {
+		in, out := &in.AdditionalPrinterColumns, &out.AdditionalPrinterColumns
+		*out = make([]CustomResourceColumnDefinition, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD

@@ -15,6 +15,7 @@ go_library(
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/validation:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go

@@ -18,15 +18,16 @@ package validation
 
 import (
 	"fmt"
-	"k8s.io/apiserver/pkg/util/webhook"
 	"reflect"
 	"strings"
 
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	genericvalidation "k8s.io/apimachinery/pkg/api/validation"
 	"k8s.io/apimachinery/pkg/util/sets"
 	validationutil "k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/apiserver/pkg/util/webhook"
 
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
 	apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation"
@@ -99,6 +100,17 @@ func ValidateUpdateCustomResourceDefinitionStatus(obj, oldObj *apiextensions.Cus
 	return allErrs
 }
 
+// ValidateCustomResourceDefinitionVersion statically validates.
+func ValidateCustomResourceDefinitionVersion(version *apiextensions.CustomResourceDefinitionVersion, fldPath *field.Path, statusEnabled bool) field.ErrorList {
+	allErrs := field.ErrorList{}
+	allErrs = append(allErrs, ValidateCustomResourceDefinitionValidation(version.Schema, statusEnabled, fldPath.Child("schema"))...)
+	allErrs = append(allErrs, ValidateCustomResourceDefinitionSubresources(version.Subresources, fldPath.Child("subresources"))...)
+	for i := range version.AdditionalPrinterColumns {
+		allErrs = append(allErrs, ValidateCustomResourceColumnDefinition(&version.AdditionalPrinterColumns[i], fldPath.Child("additionalPrinterColumns").Index(i))...)
+	}
+	return allErrs
+}
+
 // ValidateCustomResourceDefinitionSpec statically validates
 func ValidateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefinitionSpec, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
@@ -128,7 +140,32 @@ func ValidateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefi
 		if errs := validationutil.IsDNS1035Label(version.Name); len(errs) > 0 {
 			allErrs = append(allErrs, field.Invalid(fldPath.Child("versions").Index(i).Child("name"), spec.Versions[i].Name, strings.Join(errs, ",")))
 		}
+		subresources := getSubresourcesForVersion(spec, version.Name)
+		allErrs = append(allErrs, ValidateCustomResourceDefinitionVersion(&version, fldPath.Child("versions").Index(i), hasStatusEnabled(subresources))...)
+	}
+
+	// The top-level and per-version fields are mutual exclusive
+	if spec.Validation != nil && hasPerVersionSchema(spec.Versions) {
+		allErrs = append(allErrs, field.Forbidden(fldPath.Child("validation"), "top-level and per-version schemas are mutually exclusive"))
 	}
+	if spec.Subresources != nil && hasPerVersionSubresources(spec.Versions) {
+		allErrs = append(allErrs, field.Forbidden(fldPath.Child("subresources"), "top-level and per-version subresources are mutually exclusive"))
+	}
+	if len(spec.AdditionalPrinterColumns) > 0 && hasPerVersionColumns(spec.Versions) {
+		allErrs = append(allErrs, field.Forbidden(fldPath.Child("additionalPrinterColumns"), "top-level and per-version additionalPrinterColumns are mutually exclusive"))
+	}
+
+	// Per-version fields may not all be set to identical values (top-level field should be used instead)
+	if hasIdenticalPerVersionSchema(spec.Versions) {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("versions"), spec.Versions, "per-version schemas may not all be set to identical values (top-level validation should be used instead)"))
+	}
+	if hasIdenticalPerVersionSubresources(spec.Versions) {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("versions"), spec.Versions, "per-version subresources may not all be set to identical values (top-level subresources should be used instead)"))
+	}
+	if hasIdenticalPerVersionColumns(spec.Versions) {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("versions"), spec.Versions, "per-version additionalPrinterColumns may not all be set to identical values (top-level additionalPrinterColumns should be used instead)"))
+	}
+
 	if !uniqueNames {
 		allErrs = append(allErrs, field.Invalid(fldPath.Child("versions"), spec.Versions, "must contain unique version names"))
 	}
@@ -161,11 +198,7 @@ func ValidateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefi
 	allErrs = append(allErrs, ValidateCustomResourceDefinitionNames(&spec.Names, fldPath.Child("names"))...)
 
 	if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceValidation) {
-		statusEnabled := false
-		if spec.Subresources != nil && spec.Subresources.Status != nil {
-			statusEnabled = true
-		}
-		allErrs = append(allErrs, ValidateCustomResourceDefinitionValidation(spec.Validation, statusEnabled, fldPath.Child("validation"))...)
+		allErrs = append(allErrs, ValidateCustomResourceDefinitionValidation(spec.Validation, hasAnyStatusEnabled(spec), fldPath.Child("validation"))...)
 	} else if spec.Validation != nil {
 		allErrs = append(allErrs, field.Forbidden(fldPath.Child("validation"), "disabled by feature-gate CustomResourceValidation"))
 	}
@@ -177,7 +210,7 @@ func ValidateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefi
 	}
 
 	for i := range spec.AdditionalPrinterColumns {
-		if errs := ValidateCustomResourceColumnDefinition(&spec.AdditionalPrinterColumns[i], fldPath.Child("columns").Index(i)); len(errs) > 0 {
+		if errs := ValidateCustomResourceColumnDefinition(&spec.AdditionalPrinterColumns[i], fldPath.Child("additionalPrinterColumns").Index(i)); len(errs) > 0 {
 			allErrs = append(allErrs, errs...)
 		}
 	}
@@ -250,6 +283,118 @@ func ValidateCustomResourceDefinitionSpecUpdate(spec, oldSpec *apiextensions.Cus
 	return allErrs
 }
 
+// getSubresourcesForVersion returns the subresources for given version in given CRD spec.
+// NOTE That this function assumes version always exist since it's used by the validation process
+// that iterates through the existing versions.
+func getSubresourcesForVersion(crd *apiextensions.CustomResourceDefinitionSpec, version string) *apiextensions.CustomResourceSubresources {
+	if !hasPerVersionSubresources(crd.Versions) {
+		return crd.Subresources
+	}
+	for _, v := range crd.Versions {
+		if version == v.Name {
+			return v.Subresources
+		}
+	}
+	return nil
+}
+
+// hasAnyStatusEnabled returns true if given CRD spec has at least one Status Subresource set
+// among the top-level and per-version Subresources.
+func hasAnyStatusEnabled(crd *apiextensions.CustomResourceDefinitionSpec) bool {
+	if hasStatusEnabled(crd.Subresources) {
+		return true
+	}
+	for _, v := range crd.Versions {
+		if hasStatusEnabled(v.Subresources) {
+			return true
+		}
+	}
+	return false
+}
+
+// hasStatusEnabled returns true if given CRD Subresources has non-nil Status set.
+func hasStatusEnabled(subresources *apiextensions.CustomResourceSubresources) bool {
+	if subresources != nil && subresources.Status != nil {
+		return true
+	}
+	return false
+}
+
+// hasPerVersionSchema returns true if a CRD uses per-version schema.
+func hasPerVersionSchema(versions []apiextensions.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if v.Schema != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// hasPerVersionSubresources returns true if a CRD uses per-version subresources.
+func hasPerVersionSubresources(versions []apiextensions.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if v.Subresources != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// hasPerVersionColumns returns true if a CRD uses per-version columns.
+func hasPerVersionColumns(versions []apiextensions.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if len(v.AdditionalPrinterColumns) > 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// hasIdenticalPerVersionSchema returns true if a CRD sets identical non-nil values
+// to all per-version schemas
+func hasIdenticalPerVersionSchema(versions []apiextensions.CustomResourceDefinitionVersion) bool {
+	if len(versions) == 0 {
+		return false
+	}
+	value := versions[0].Schema
+	for _, v := range versions {
+		if v.Schema == nil || !apiequality.Semantic.DeepEqual(v.Schema, value) {
+			return false
+		}
+	}
+	return true
+}
+
+// hasIdenticalPerVersionSubresources returns true if a CRD sets identical non-nil values
+// to all per-version subresources
+func hasIdenticalPerVersionSubresources(versions []apiextensions.CustomResourceDefinitionVersion) bool {
+	if len(versions) == 0 {
+		return false
+	}
+	value := versions[0].Subresources
+	for _, v := range versions {
+		if v.Subresources == nil || !apiequality.Semantic.DeepEqual(v.Subresources, value) {
+			return false
+		}
+	}
+	return true
+}
+
+// hasIdenticalPerVersionColumns returns true if a CRD sets identical non-nil values
+// to all per-version columns
+func hasIdenticalPerVersionColumns(versions []apiextensions.CustomResourceDefinitionVersion) bool {
+	if len(versions) == 0 {
+		return false
+	}
+	value := versions[0].AdditionalPrinterColumns
+	for _, v := range versions {
+		if len(v.AdditionalPrinterColumns) == 0 || !apiequality.Semantic.DeepEqual(v.AdditionalPrinterColumns, value) {
+			return false
+		}
+	}
+	return true
+}
+
 // ValidateCustomResourceDefinitionStatus statically validates
 func ValidateCustomResourceDefinitionStatus(status *apiextensions.CustomResourceDefinitionStatus, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go

@@ -594,6 +594,55 @@ func TestValidateCustomResourceDefinition(t *testing.T) {
 			},
 			errors: []validationMatch{},
 		},
+		{
+			name: "per-version fields may not all be set to identical values (top-level field should be used instead)",
+			resource: &apiextensions.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"},
+				Spec: apiextensions.CustomResourceDefinitionSpec{
+					Group:   "group.com",
+					Version: "version",
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: true,
+							Schema: &apiextensions.CustomResourceValidation{
+								OpenAPIV3Schema: validValidationSchema,
+							},
+							Subresources:             &apiextensions.CustomResourceSubresources{},
+							AdditionalPrinterColumns: []apiextensions.CustomResourceColumnDefinition{{Name: "Alpha", Type: "string", JSONPath: ".spec.alpha"}},
+						},
+						{
+							Name:    "version2",
+							Served:  true,
+							Storage: false,
+							Schema: &apiextensions.CustomResourceValidation{
+								OpenAPIV3Schema: validValidationSchema,
+							},
+							Subresources:             &apiextensions.CustomResourceSubresources{},
+							AdditionalPrinterColumns: []apiextensions.CustomResourceColumnDefinition{{Name: "Alpha", Type: "string", JSONPath: ".spec.alpha"}},
+						},
+					},
+					Scope: apiextensions.NamespaceScoped,
+					Names: apiextensions.CustomResourceDefinitionNames{
+						Plural:   "plural",
+						Singular: "singular",
+						Kind:     "Plural",
+						ListKind: "PluralList",
+					},
+				},
+				Status: apiextensions.CustomResourceDefinitionStatus{
+					StoredVersions: []string{"version"},
+				},
+			},
+			errors: []validationMatch{
+				// Per-version schema/subresources/columns may not all be set to identical values.
+				// Note that the test will fail if we de-duplicate the expected errors below.
+				invalid("spec", "versions"),
+				invalid("spec", "versions"),
+				invalid("spec", "versions"),
+			},
+		},
 	}
 
 	for _, tc := range tests {
@@ -1003,6 +1052,87 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) {
 				immutable("spec", "names", "plural"),
 			},
 		},
+		{
+			name: "top-level and per-version fields are mutually exclusive",
+			old: &apiextensions.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            "plural.group.com",
+					ResourceVersion: "42",
+				},
+				Spec: apiextensions.CustomResourceDefinitionSpec{
+					Group:   "group.com",
+					Version: "version",
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:         "version",
+							Served:       true,
+							Storage:      true,
+							Subresources: &apiextensions.CustomResourceSubresources{},
+						},
+						{
+							Name:    "version2",
+							Served:  true,
+							Storage: false,
+						},
+					},
+					Scope: apiextensions.NamespaceScoped,
+					Names: apiextensions.CustomResourceDefinitionNames{
+						Plural:   "plural",
+						Singular: "singular",
+						Kind:     "Plural",
+						ListKind: "PluralList",
+					},
+				},
+				Status: apiextensions.CustomResourceDefinitionStatus{
+					StoredVersions: []string{"version"},
+				},
+			},
+			resource: &apiextensions.CustomResourceDefinition{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            "plural.group.com",
+					ResourceVersion: "42",
+				},
+				Spec: apiextensions.CustomResourceDefinitionSpec{
+					Group:   "group.com",
+					Version: "version",
+					Versions: []apiextensions.CustomResourceDefinitionVersion{
+						{
+							Name:    "version",
+							Served:  true,
+							Storage: true,
+						},
+						{
+							Name:    "version2",
+							Served:  true,
+							Storage: false,
+							Schema: &apiextensions.CustomResourceValidation{
+								OpenAPIV3Schema: validValidationSchema,
+							},
+							Subresources:             &apiextensions.CustomResourceSubresources{},
+							AdditionalPrinterColumns: []apiextensions.CustomResourceColumnDefinition{{Name: "Alpha", Type: "string", JSONPath: ".spec.alpha"}},
+						},
+					},
+					Validation: &apiextensions.CustomResourceValidation{
+						OpenAPIV3Schema: validValidationSchema,
+					},
+					Subresources: &apiextensions.CustomResourceSubresources{},
+					Scope:        apiextensions.NamespaceScoped,
+					Names: apiextensions.CustomResourceDefinitionNames{
+						Plural:   "plural",
+						Singular: "singular",
+						Kind:     "Plural",
+						ListKind: "PluralList",
+					},
+				},
+				Status: apiextensions.CustomResourceDefinitionStatus{
+					StoredVersions: []string{"version"},
+				},
+			},
+			errors: []validationMatch{
+				forbidden("spec", "validation"),
+				forbidden("spec", "subresources"),
+			},
+		},
 	}
 
 	for _, tc := range tests {
@@ -1090,36 +1220,7 @@ func TestValidateCustomResourceDefinitionValidation(t *testing.T) {
 		{
 			name: "all allowed fields at the root of the schema with status",
 			input: apiextensions.CustomResourceValidation{
-				OpenAPIV3Schema: &apiextensions.JSONSchemaProps{
-					Description:      "This is a description",
-					Type:             "object",
-					Format:           "date-time",
-					Title:            "This is a title",
-					Maximum:          float64Ptr(10),
-					ExclusiveMaximum: true,
-					Minimum:          float64Ptr(5),
-					ExclusiveMinimum: true,
-					MaxLength:        int64Ptr(10),
-					MinLength:        int64Ptr(5),
-					Pattern:          "^[a-z]$",
-					MaxItems:         int64Ptr(10),
-					MinItems:         int64Ptr(5),
-					MultipleOf:       float64Ptr(3),
-					Required:         []string{"spec", "status"},
-					Items: &apiextensions.JSONSchemaPropsOrArray{
-						Schema: &apiextensions.JSONSchemaProps{
-							Description: "This is a schema nested under Items",
-						},
-					},
-					Properties: map[string]apiextensions.JSONSchemaProps{
-						"spec":   {},
-						"status": {},
-					},
-					ExternalDocs: &apiextensions.ExternalDocumentation{
-						Description: "This is an external documentation description",
-					},
-					Example: &example,
-				},
+				OpenAPIV3Schema: validValidationSchema,
 			},
 			statusEnabled: true,
 			wantError:     false,
@@ -1139,6 +1240,37 @@ func TestValidateCustomResourceDefinitionValidation(t *testing.T) {
 
 var example = apiextensions.JSON(`"This is an example"`)
 
+var validValidationSchema = &apiextensions.JSONSchemaProps{
+	Description:      "This is a description",
+	Type:             "object",
+	Format:           "date-time",
+	Title:            "This is a title",
+	Maximum:          float64Ptr(10),
+	ExclusiveMaximum: true,
+	Minimum:          float64Ptr(5),
+	ExclusiveMinimum: true,
+	MaxLength:        int64Ptr(10),
+	MinLength:        int64Ptr(5),
+	Pattern:          "^[a-z]$",
+	MaxItems:         int64Ptr(10),
+	MinItems:         int64Ptr(5),
+	MultipleOf:       float64Ptr(3),
+	Required:         []string{"spec", "status"},
+	Items: &apiextensions.JSONSchemaPropsOrArray{
+		Schema: &apiextensions.JSONSchemaProps{
+			Description: "This is a schema nested under Items",
+		},
+	},
+	Properties: map[string]apiextensions.JSONSchemaProps{
+		"spec":   {},
+		"status": {},
+	},
+	ExternalDocs: &apiextensions.ExternalDocumentation{
+		Description: "This is an external documentation description",
+	},
+	Example: &example,
+}
+
 func float64Ptr(f float64) *float64 {
 	return &f
 }

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/zz_generated.deepcopy.go

@@ -182,7 +182,9 @@ func (in *CustomResourceDefinitionSpec) DeepCopyInto(out *CustomResourceDefiniti
 	if in.Versions != nil {
 		in, out := &in.Versions, &out.Versions
 		*out = make([]CustomResourceDefinitionVersion, len(*in))
-		copy(*out, *in)
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
 	}
 	if in.AdditionalPrinterColumns != nil {
 		in, out := &in.AdditionalPrinterColumns, &out.AdditionalPrinterColumns
@@ -239,6 +241,21 @@ func (in *CustomResourceDefinitionStatus) DeepCopy() *CustomResourceDefinitionSt
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CustomResourceDefinitionVersion) DeepCopyInto(out *CustomResourceDefinitionVersion) {
 	*out = *in
+	if in.Schema != nil {
+		in, out := &in.Schema, &out.Schema
+		*out = new(CustomResourceValidation)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Subresources != nil {
+		in, out := &in.Subresources, &out.Subresources
+		*out = new(CustomResourceSubresources)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.AdditionalPrinterColumns != nil {
+		in, out := &in.AdditionalPrinterColumns, &out.AdditionalPrinterColumns
+		*out = make([]CustomResourceColumnDefinition, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD

@@ -13,6 +13,7 @@ go_library(
         "customresource_discovery.go",
         "customresource_discovery_controller.go",
         "customresource_handler.go",
+        "helpers.go",
     ],
     importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver",
     importpath = "k8s.io/apiextensions-apiserver/pkg/apiserver",

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go

@@ -136,7 +136,11 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 			Categories:   crd.Status.AcceptedNames.Categories,
 		})
 
-		if crd.Spec.Subresources != nil && crd.Spec.Subresources.Status != nil {
+		subresources, err := getSubresourcesForVersion(crd, version.Version)
+		if err != nil {
+			return err
+		}
+		if subresources != nil && subresources.Status != nil {
 			apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{
 				Name:       crd.Status.AcceptedNames.Plural + "/status",
 				Namespaced: crd.Spec.Scope == apiextensions.NamespaceScoped,
@@ -145,7 +149,7 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 			})
 		}
 
-		if crd.Spec.Subresources != nil && crd.Spec.Subresources.Scale != nil {
+		if subresources != nil && subresources.Scale != nil {
 			apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{
 				Group:      autoscaling.GroupName,
 				Version:    "v1",

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go

@@ -220,10 +220,16 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	}
 
 	var handler http.HandlerFunc
+	subresources, err := getSubresourcesForVersion(crd, requestInfo.APIVersion)
+	if err != nil {
+		utilruntime.HandleError(err)
+		http.Error(w, "the server could not properly serve the CR subresources", http.StatusInternalServerError)
+		return
+	}
 	switch {
-	case subresource == "status" && crd.Spec.Subresources != nil && crd.Spec.Subresources.Status != nil:
+	case subresource == "status" && subresources != nil && subresources.Status != nil:
 		handler = r.serveStatus(w, req, requestInfo, crdInfo, terminating, supportedTypes)
-	case subresource == "scale" && crd.Spec.Subresources != nil && crd.Spec.Subresources.Scale != nil:
+	case subresource == "scale" && subresources != nil && subresources.Scale != nil:
 		handler = r.serveScale(w, req, requestInfo, crdInfo, terminating, supportedTypes)
 	case len(subresource) == 0:
 		handler = r.serveResource(w, req, requestInfo, crdInfo, terminating, supportedTypes)
@@ -443,19 +449,28 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource
 		typer := newUnstructuredObjectTyper(parameterScheme)
 		creator := unstructuredCreator{}
 
-		validator, _, err := apiservervalidation.NewSchemaValidator(crd.Spec.Validation)
+		validationSchema, err := getSchemaForVersion(crd, v.Name)
+		if err != nil {
+			utilruntime.HandleError(err)
+			return nil, fmt.Errorf("the server could not properly serve the CR schema")
+		}
+		validator, _, err := apiservervalidation.NewSchemaValidator(validationSchema)
 		if err != nil {
 			return nil, err
 		}
 
 		var statusSpec *apiextensions.CustomResourceSubresourceStatus
 		var statusValidator *validate.SchemaValidator
-		if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && crd.Spec.Subresources != nil && crd.Spec.Subresources.Status != nil {
-			statusSpec = crd.Spec.Subresources.Status
-
+		subresources, err := getSubresourcesForVersion(crd, v.Name)
+		if err != nil {
+			utilruntime.HandleError(err)
+			return nil, fmt.Errorf("the server could not properly serve the CR subresources")
+		}
+		if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && subresources != nil && subresources.Status != nil {
+			statusSpec = subresources.Status
 			// for the status subresource, validate only against the status schema
-			if crd.Spec.Validation != nil && crd.Spec.Validation.OpenAPIV3Schema != nil && crd.Spec.Validation.OpenAPIV3Schema.Properties != nil {
-				if statusSchema, ok := crd.Spec.Validation.OpenAPIV3Schema.Properties["status"]; ok {
+			if validationSchema != nil && validationSchema.OpenAPIV3Schema != nil && validationSchema.OpenAPIV3Schema.Properties != nil {
+				if statusSchema, ok := validationSchema.OpenAPIV3Schema.Properties["status"]; ok {
 					openapiSchema := &spec.Schema{}
 					if err := apiservervalidation.ConvertJSONSchemaProps(&statusSchema, openapiSchema); err != nil {
 						return nil, err
@@ -466,11 +481,16 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource
 		}
 
 		var scaleSpec *apiextensions.CustomResourceSubresourceScale
-		if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && crd.Spec.Subresources != nil && crd.Spec.Subresources.Scale != nil {
-			scaleSpec = crd.Spec.Subresources.Scale
+		if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) && subresources != nil && subresources.Scale != nil {
+			scaleSpec = subresources.Scale
 		}
 
-		table, err := tableconvertor.New(crd.Spec.AdditionalPrinterColumns)
+		columns, err := getColumnsForVersion(crd, v.Name)
+		if err != nil {
+			utilruntime.HandleError(err)
+			return nil, fmt.Errorf("the server could not properly serve the CR columns")
+		}
+		table, err := tableconvertor.New(columns)
 		if err != nil {
 			glog.V(2).Infof("The CRD for %v has an invalid printer specification, falling back to default printing: %v", kind, err)
 		}

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/helpers.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"fmt"
+
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+)
+
+// getSchemaForVersion returns the validation schema for given version in given CRD.
+func getSchemaForVersion(crd *apiextensions.CustomResourceDefinition, version string) (*apiextensions.CustomResourceValidation, error) {
+	if !hasPerVersionSchema(crd.Spec.Versions) {
+		return crd.Spec.Validation, nil
+	}
+	if crd.Spec.Validation != nil {
+		return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version schemas must be mutual exclusive", crd.Name, version)
+	}
+	for _, v := range crd.Spec.Versions {
+		if version == v.Name {
+			return v.Schema, nil
+		}
+	}
+	return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name)
+}
+
+// getSubresourcesForVersion returns the subresources for given version in given CRD.
+func getSubresourcesForVersion(crd *apiextensions.CustomResourceDefinition, version string) (*apiextensions.CustomResourceSubresources, error) {
+	if !hasPerVersionSubresources(crd.Spec.Versions) {
+		return crd.Spec.Subresources, nil
+	}
+	if crd.Spec.Subresources != nil {
+		return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version subresources must be mutual exclusive", crd.Name, version)
+	}
+	for _, v := range crd.Spec.Versions {
+		if version == v.Name {
+			return v.Subresources, nil
+		}
+	}
+	return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name)
+}
+
+// getColumnsForVersion returns the columns for given version in given CRD.
+// NOTE: the newly logically-defaulted columns is not pointing to the original CRD object.
+// One cannot mutate the original CRD columns using the logically-defaulted columns. Please iterate through
+// the original CRD object instead.
+func getColumnsForVersion(crd *apiextensions.CustomResourceDefinition, version string) ([]apiextensions.CustomResourceColumnDefinition, error) {
+	if !hasPerVersionColumns(crd.Spec.Versions) {
+		return serveDefaultColumnsIfEmpty(crd.Spec.AdditionalPrinterColumns), nil
+	}
+	if len(crd.Spec.AdditionalPrinterColumns) > 0 {
+		return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version additionalPrinterColumns must be mutual exclusive", crd.Name, version)
+	}
+	for _, v := range crd.Spec.Versions {
+		if version == v.Name {
+			return serveDefaultColumnsIfEmpty(v.AdditionalPrinterColumns), nil
+		}
+	}
+	return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name)
+}
+
+// serveDefaultColumnsIfEmpty applies logically defaulting to columns, if the input columns is empty.
+// NOTE: in this way, the newly logically-defaulted columns is not pointing to the original CRD object.
+// One cannot mutate the original CRD columns using the logically-defaulted columns. Please iterate through
+// the original CRD object instead.
+func serveDefaultColumnsIfEmpty(columns []apiextensions.CustomResourceColumnDefinition) []apiextensions.CustomResourceColumnDefinition {
+	if len(columns) > 0 {
+		return columns
+	}
+	return []apiextensions.CustomResourceColumnDefinition{
+		{Name: "Age", Type: "date", Description: swaggerMetadataDescriptions["creationTimestamp"], JSONPath: ".metadata.creationTimestamp"},
+	}
+}
+
+// hasPerVersionSchema returns true if a CRD uses per-version schema.
+func hasPerVersionSchema(versions []apiextensions.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if v.Schema != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// hasPerVersionSubresources returns true if a CRD uses per-version subresources.
+func hasPerVersionSubresources(versions []apiextensions.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if v.Subresources != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// hasPerVersionColumns returns true if a CRD uses per-version columns.
+func hasPerVersionColumns(versions []apiextensions.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if len(v.AdditionalPrinterColumns) > 0 {
+			return true
+		}
+	}
+	return false
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go

@@ -41,7 +41,7 @@ const (
 	// CustomResourceSubresources defines the subresources for CustomResources
 	CustomResourceSubresources utilfeature.Feature = "CustomResourceSubresources"
 
-	// owner: @mbohlool
+	// owner: @mbohlool, @roycaihw
 	// alpha: v1.13
 	//
 	// CustomResourceWebhookConversion defines the webhook conversion for Custom Resources.

staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go

@@ -57,9 +57,25 @@ func (strategy) PrepareForCreate(ctx context.Context, obj runtime.Object) {
 	// if the feature gate is disabled, drop the feature.
 	if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceValidation) {
 		crd.Spec.Validation = nil
+		for i := range crd.Spec.Versions {
+			crd.Spec.Versions[i].Schema = nil
+		}
 	}
 	if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) {
 		crd.Spec.Subresources = nil
+		for i := range crd.Spec.Versions {
+			crd.Spec.Versions[i].Subresources = nil
+		}
+	}
+	// On CREATE, if the CustomResourceWebhookConversion feature gate is off, we auto-clear
+	// the per-version fields. This is to be consistent with the other built-in types, as the
+	// apiserver drops unknown fields.
+	if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) {
+		for i := range crd.Spec.Versions {
+			crd.Spec.Versions[i].Schema = nil
+			crd.Spec.Versions[i].Subresources = nil
+			crd.Spec.Versions[i].AdditionalPrinterColumns = nil
+		}
 	}
 	if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) && crd.Spec.Conversion != nil {
 		crd.Spec.Conversion.WebhookClientConfig = nil
@@ -96,10 +112,36 @@ func (strategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
 	if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceValidation) {
 		newCRD.Spec.Validation = nil
 		oldCRD.Spec.Validation = nil
+		for i := range newCRD.Spec.Versions {
+			newCRD.Spec.Versions[i].Schema = nil
+		}
+		for i := range oldCRD.Spec.Versions {
+			oldCRD.Spec.Versions[i].Schema = nil
+		}
 	}
 	if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceSubresources) {
 		newCRD.Spec.Subresources = nil
 		oldCRD.Spec.Subresources = nil
+		for i := range newCRD.Spec.Versions {
+			newCRD.Spec.Versions[i].Subresources = nil
+		}
+		for i := range oldCRD.Spec.Versions {
+			oldCRD.Spec.Versions[i].Subresources = nil
+		}
+	}
+
+	// On UPDATE, if the CustomResourceWebhookConversion feature gate is off, we auto-clear
+	// the per-version fields if the old CRD doesn't use per-version fields already.
+	// This is to be consistent with the other built-in types, as the apiserver drops unknown
+	// fields. If the old CRD already uses per-version fields, the CRD is allowed to continue
+	// use per-version fields.
+	if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) &&
+		!hasPerVersionField(oldCRD.Spec.Versions) {
+		for i := range newCRD.Spec.Versions {
+			newCRD.Spec.Versions[i].Schema = nil
+			newCRD.Spec.Versions[i].Subresources = nil
+			newCRD.Spec.Versions[i].AdditionalPrinterColumns = nil
+		}
 	}
 	if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) && newCRD.Spec.Conversion != nil {
 		if oldCRD.Spec.Conversion == nil || newCRD.Spec.Conversion.WebhookClientConfig == nil {
@@ -117,6 +159,16 @@ func (strategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
 	}
 }
 
+// hasPerVersionField returns true if a CRD uses per-version schema/subresources/columns fields.
+func hasPerVersionField(versions []apiextensions.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if v.Schema != nil || v.Subresources != nil || len(v.AdditionalPrinterColumns) > 0 {
+			return true
+		}
+	}
+	return false
+}
+
 // Validate validates a new CustomResourceDefinition.
 func (strategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList {
 	return validation.ValidateCustomResourceDefinition(obj.(*apiextensions.CustomResourceDefinition))

staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD

@@ -26,6 +26,7 @@ go_test(
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
@@ -40,6 +41,8 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
         "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//vendor/github.com/coreos/etcd/clientv3:go_default_library",

staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/resources.go

@@ -370,13 +370,13 @@ func DeleteCustomResourceDefinition(crd *apiextensionsv1beta1.CustomResourceDefi
 	return nil
 }
 
-// CreateNewScaleClient returns a scale client.
-func CreateNewScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, config *rest.Config) (scale.ScalesGetter, error) {
+// CreateNewVersionedScaleClient returns a scale client.
+func CreateNewVersionedScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, config *rest.Config, version string) (scale.ScalesGetter, error) {
 	discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
 	if err != nil {
 		return nil, err
 	}
-	groupResource, err := discoveryClient.ServerResourcesForGroupVersion(crd.Spec.Group + "/" + crd.Spec.Version)
+	groupResource, err := discoveryClient.ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version)
 	if err != nil {
 		return nil, err
 	}
@@ -386,12 +386,12 @@ func CreateNewScaleClient(crd *apiextensionsv1beta1.CustomResourceDefinition, co
 			Group: metav1.APIGroup{
 				Name: crd.Spec.Group,
 				Versions: []metav1.GroupVersionForDiscovery{
-					{Version: crd.Spec.Version},
+					{Version: version},
 				},
-				PreferredVersion: metav1.GroupVersionForDiscovery{Version: crd.Spec.Version},
+				PreferredVersion: metav1.GroupVersionForDiscovery{Version: version},
 			},
 			VersionedResources: map[string][]metav1.APIResource{
-				crd.Spec.Version: groupResource.APIResources,
+				version: groupResource.APIResources,
 			},
 		},
 	}

staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go

@@ -30,6 +30,8 @@ import (
 	"k8s.io/client-go/dynamic"
 )
 
+var swaggerMetadataDescriptions = metav1.ObjectMeta{}.SwaggerDoc()
+
 func instantiateCustomResource(t *testing.T, instanceToCreate *unstructured.Unstructured, client dynamic.ResourceInterface, definition *apiextensionsv1beta1.CustomResourceDefinition) (*unstructured.Unstructured, error) {
 	return instantiateVersionedCustomResource(t, instanceToCreate, client, definition, definition.Spec.Versions[0].Name)
 }
@@ -92,3 +94,97 @@ func updateCustomResourceDefinitionWithRetry(client clientset.Interface, name st
 	}
 	return nil, fmt.Errorf("too many retries after conflicts updating CustomResourceDefinition %q", name)
 }
+
+// getSchemaForVersion returns the validation schema for given version in given CRD.
+func getSchemaForVersion(crd *apiextensionsv1beta1.CustomResourceDefinition, version string) (*apiextensionsv1beta1.CustomResourceValidation, error) {
+	if !hasPerVersionSchema(crd.Spec.Versions) {
+		return crd.Spec.Validation, nil
+	}
+	if crd.Spec.Validation != nil {
+		return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version schemas must be mutual exclusive", crd.Name, version)
+	}
+	for _, v := range crd.Spec.Versions {
+		if version == v.Name {
+			return v.Schema, nil
+		}
+	}
+	return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name)
+}
+
+// getSubresourcesForVersion returns the subresources for given version in given CRD.
+func getSubresourcesForVersion(crd *apiextensionsv1beta1.CustomResourceDefinition, version string) (*apiextensionsv1beta1.CustomResourceSubresources, error) {
+	if !hasPerVersionSubresources(crd.Spec.Versions) {
+		return crd.Spec.Subresources, nil
+	}
+	if crd.Spec.Subresources != nil {
+		return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version subresources must be mutual exclusive", crd.Name, version)
+	}
+	for _, v := range crd.Spec.Versions {
+		if version == v.Name {
+			return v.Subresources, nil
+		}
+	}
+	return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name)
+}
+
+// getColumnsForVersion returns the columns for given version in given CRD.
+// NOTE: the newly logically-defaulted columns is not pointing to the original CRD object.
+// One cannot mutate the original CRD columns using the logically-defaulted columns. Please iterate through
+// the original CRD object instead.
+func getColumnsForVersion(crd *apiextensionsv1beta1.CustomResourceDefinition, version string) ([]apiextensionsv1beta1.CustomResourceColumnDefinition, error) {
+	if !hasPerVersionColumns(crd.Spec.Versions) {
+		return serveDefaultColumnsIfEmpty(crd.Spec.AdditionalPrinterColumns), nil
+	}
+	if len(crd.Spec.AdditionalPrinterColumns) > 0 {
+		return nil, fmt.Errorf("malformed CustomResourceDefinition %s version %s: top-level and per-version additionalPrinterColumns must be mutual exclusive", crd.Name, version)
+	}
+	for _, v := range crd.Spec.Versions {
+		if version == v.Name {
+			return serveDefaultColumnsIfEmpty(v.AdditionalPrinterColumns), nil
+		}
+	}
+	return nil, fmt.Errorf("version %s not found in CustomResourceDefinition: %v", version, crd.Name)
+}
+
+// serveDefaultColumnsIfEmpty applies logically defaulting to columns, if the input columns is empty.
+// NOTE: in this way, the newly logically-defaulted columns is not pointing to the original CRD object.
+// One cannot mutate the original CRD columns using the logically-defaulted columns. Please iterate through
+// the original CRD object instead.
+func serveDefaultColumnsIfEmpty(columns []apiextensionsv1beta1.CustomResourceColumnDefinition) []apiextensionsv1beta1.CustomResourceColumnDefinition {
+	if len(columns) > 0 {
+		return columns
+	}
+	return []apiextensionsv1beta1.CustomResourceColumnDefinition{
+		{Name: "Age", Type: "date", Description: swaggerMetadataDescriptions["creationTimestamp"], JSONPath: ".metadata.creationTimestamp"},
+	}
+}
+
+// hasPerVersionSchema returns true if a CRD uses per-version schema.
+func hasPerVersionSchema(versions []apiextensionsv1beta1.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if v.Schema != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// hasPerVersionSubresources returns true if a CRD uses per-version subresources.
+func hasPerVersionSubresources(versions []apiextensionsv1beta1.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if v.Subresources != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// hasPerVersionColumns returns true if a CRD uses per-version columns.
+func hasPerVersionColumns(versions []apiextensionsv1beta1.CustomResourceDefinitionVersion) bool {
+	for _, v := range versions {
+		if len(v.AdditionalPrinterColumns) > 0 {
+			return true
+		}
+	}
+	return false
+}

staging/src/k8s.io/apiextensions-apiserver/test/integration/yaml_test.go

@@ -27,10 +27,13 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	"k8s.io/client-go/dynamic"
 
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features"
 	"k8s.io/apiextensions-apiserver/test/integration/fixtures"
 )
 
@@ -354,6 +357,7 @@ values:
 }
 
 func TestYAMLSubresource(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)()
 	tearDown, config, _, err := fixtures.StartDefaultServer(t)
 	if err != nil {
 		t.Fatal(err)
@@ -369,7 +373,7 @@ func TestYAMLSubresource(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	noxuDefinition := NewNoxuSubresourcesCRD(apiextensionsv1beta1.ClusterScoped)
+	noxuDefinition := NewNoxuSubresourcesCRDs(apiextensionsv1beta1.ClusterScoped)[0]
 	noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
 	if err != nil {
 		t.Fatal(err)