Merge pull request #72942 from caesarxuchao/expose-storage-version-hash

cmd/kube-apiserver/app/BUILD

@@ -39,6 +39,7 @@ go_library(
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",

cmd/kube-apiserver/app/aggregator.go

@@ -30,6 +30,7 @@ import (
 
 	apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/admission"
@@ -79,6 +80,7 @@ func createAggregatorConfig(
 	etcdOptions := *commandOptions.Etcd
 	etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
 	etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion)
+	etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName})
 	genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions}
 
 	// override MergedResourceConfig with aggregator defaults and registry

cmd/kube-apiserver/app/apiextensions.go

@@ -23,6 +23,8 @@ import (
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
 	apiextensionsoptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/features"
 	genericapiserver "k8s.io/apiserver/pkg/server"
@@ -61,6 +63,7 @@ func createAPIExtensionsConfig(
 	etcdOptions := *commandOptions.Etcd
 	etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
 	etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion)
+	etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName})
 	genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions}
 
 	// override MergedResourceConfig with apiextensions defaults and registry

pkg/master/BUILD

@@ -141,10 +141,13 @@ go_test(
     deps = [
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/api/testapi:go_default_library",
+        "//pkg/apis/batch:go_default_library",
         "//pkg/apis/core:go_default_library",
+        "//pkg/apis/storage:go_default_library",
         "//pkg/generated/openapi:go_default_library",
         "//pkg/kubelet/client:go_default_library",
         "//pkg/master/reconcilers:go_default_library",
+        "//pkg/master/storageversionhashdata:go_default_library",
         "//pkg/registry/certificates/rest:go_default_library",
         "//pkg/registry/core/rest:go_default_library",
         "//pkg/registry/registrytest:go_default_library",
@@ -154,16 +157,23 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/api/apitesting/naming:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/features:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/resourceconfig:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
+        "//staging/src/k8s.io/client-go/discovery:go_default_library",
         "//staging/src/k8s.io/client-go/informers:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
@@ -191,6 +201,7 @@ filegroup(
         "//pkg/master/controller/crdregistration:all-srcs",
         "//pkg/master/ports:all-srcs",
         "//pkg/master/reconcilers:all-srcs",
+        "//pkg/master/storageversionhashdata:all-srcs",
         "//pkg/master/tunneler:all-srcs",
     ],
     tags = ["automanaged"],

pkg/master/master_test.go

@@ -31,22 +31,32 @@ import (
 	certificatesapiv1beta1 "k8s.io/api/certificates/v1beta1"
 	apiv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	"k8s.io/apiserver/pkg/features"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/options"
+	"k8s.io/apiserver/pkg/server/resourceconfig"
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
 	etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
+	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/api/testapi"
+	"k8s.io/kubernetes/pkg/apis/batch"
 	api "k8s.io/kubernetes/pkg/apis/core"
+	apisstorage "k8s.io/kubernetes/pkg/apis/storage"
 	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
 	"k8s.io/kubernetes/pkg/master/reconcilers"
+	"k8s.io/kubernetes/pkg/master/storageversionhashdata"
 	certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest"
 	corerest "k8s.io/kubernetes/pkg/registry/core/rest"
 	"k8s.io/kubernetes/pkg/registry/registrytest"
@@ -70,6 +80,14 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion
 	}
 
 	resourceEncoding := serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Scheme)
+	// This configures the testing master the same way the real master is
+	// configured. The storage versions of these resources are different
+	// from the storage versions of other resources in their group.
+	resourceEncodingOverrides := []schema.GroupVersionResource{
+		batch.Resource("cronjobs").WithVersion("v1beta1"),
+		apisstorage.Resource("volumeattachments").WithVersion("v1beta1"),
+	}
+	resourceEncoding = resourceconfig.MergeResourceEncodingConfigs(resourceEncoding, resourceEncodingOverrides)
 	storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, testapi.StorageMediaType(), legacyscheme.Codecs, resourceEncoding, DefaultAPIResourceConfigSource(), nil)
 
 	etcdOptions := options.NewEtcdOptions(storageConfig)
@@ -81,12 +99,12 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion
 	}
 
 	kubeVersion := kubeversion.Get()
+	config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
 	config.GenericConfig.Version = &kubeVersion
 	config.ExtraConfig.StorageFactory = storageFactory
 	config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}}
 	config.GenericConfig.PublicAddress = net.ParseIP("192.168.10.4")
 	config.GenericConfig.LegacyAPIGroupPrefixes = sets.NewString("/api")
-	config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}}
 	config.ExtraConfig.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250}
 	config.ExtraConfig.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{
 		DialContext:     func(ctx context.Context, network, addr string) (net.Conn, error) { return nil, nil },
@@ -363,6 +381,112 @@ func TestAPIVersionOfDiscoveryEndpoints(t *testing.T) {
 
 }
 
+// This test doesn't cover the apiregistration and apiextensions group, as they are installed by other apiservers.
+func TestStorageVersionHashes(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionHash, true)()
+	master, etcdserver, _, _ := newMaster(t)
+	defer etcdserver.Terminate(t)
+
+	server := httptest.NewServer(master.GenericAPIServer.Handler.GoRestfulContainer.ServeMux)
+
+	c := &restclient.Config{
+		Host:          server.URL,
+		APIPath:       "/api",
+		ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs},
+	}
+	discover := discovery.NewDiscoveryClientForConfigOrDie(c)
+	all, err := discover.ServerResources()
+	if err != nil {
+		t.Error(err)
+	}
+	var count int
+	for _, g := range all {
+		for _, r := range g.APIResources {
+			if strings.Contains(r.Name, "/") ||
+				storageversionhashdata.NoStorageVersionHash.Has(g.GroupVersion+"/"+r.Name) {
+				if r.StorageVersionHash != "" {
+					t.Errorf("expect resource %s/%s to have empty storageVersionHash, got hash %q", g.GroupVersion, r.Name, r.StorageVersionHash)
+				}
+				continue
+			}
+			if r.StorageVersionHash == "" {
+				t.Errorf("expect the storageVersionHash of %s/%s to exist", g.GroupVersion, r.Name)
+				continue
+			}
+			// Uncomment the following line if you want to update storageversionhash/data.go
+			// fmt.Printf("\"%s/%s\": \"%s\",\n", g.GroupVersion, r.Name, r.StorageVersionHash)
+			expected := storageversionhashdata.GVRToStorageVersionHash[g.GroupVersion+"/"+r.Name]
+			if r.StorageVersionHash != expected {
+				t.Errorf("expect the storageVersionHash of %s/%s to be %q, got %q", g.GroupVersion, r.Name, expected, r.StorageVersionHash)
+			}
+			count++
+		}
+	}
+	if count != len(storageversionhashdata.GVRToStorageVersionHash) {
+		t.Errorf("please remove the redundant entries from GVRToStorageVersionHash")
+	}
+}
+
+func TestStorageVersionHashEqualities(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionHash, true)()
+	master, etcdserver, _, assert := newMaster(t)
+	defer etcdserver.Terminate(t)
+
+	server := httptest.NewServer(master.GenericAPIServer.Handler.GoRestfulContainer.ServeMux)
+
+	// Test 1: extensions/v1beta1/replicasets and apps/v1/replicasets have
+	// the same storage version hash.
+	resp, err := http.Get(server.URL + "/apis/extensions/v1beta1")
+	assert.Empty(err)
+	extList := metav1.APIResourceList{}
+	assert.NoError(decodeResponse(resp, &extList))
+	var extReplicasetHash, appsReplicasetHash string
+	for _, r := range extList.APIResources {
+		if r.Name == "replicasets" {
+			extReplicasetHash = r.StorageVersionHash
+		}
+	}
+	assert.NotEmpty(extReplicasetHash)
+
+	resp, err = http.Get(server.URL + "/apis/apps/v1")
+	assert.Empty(err)
+	appsList := metav1.APIResourceList{}
+	assert.NoError(decodeResponse(resp, &appsList))
+	for _, r := range appsList.APIResources {
+		if r.Name == "replicasets" {
+			appsReplicasetHash = r.StorageVersionHash
+		}
+	}
+	assert.Equal(extReplicasetHash, appsReplicasetHash)
+
+	// Test 2: batch/v1/jobs and batch/v1beta1/cronjobs have different
+	// storage version hashes.
+	resp, err = http.Get(server.URL + "/apis/batch/v1")
+	assert.Empty(err)
+	batchv1 := metav1.APIResourceList{}
+	assert.NoError(decodeResponse(resp, &batchv1))
+	var jobsHash string
+	for _, r := range batchv1.APIResources {
+		if r.Name == "jobs" {
+			jobsHash = r.StorageVersionHash
+		}
+	}
+	assert.NotEmpty(jobsHash)
+
+	resp, err = http.Get(server.URL + "/apis/batch/v1beta1")
+	assert.Empty(err)
+	batchv1beta1 := metav1.APIResourceList{}
+	assert.NoError(decodeResponse(resp, &batchv1beta1))
+	var cronjobsHash string
+	for _, r := range batchv1beta1.APIResources {
+		if r.Name == "cronjobs" {
+			cronjobsHash = r.StorageVersionHash
+		}
+	}
+	assert.NotEmpty(cronjobsHash)
+	assert.NotEqual(jobsHash, cronjobsHash)
+}
+
 func TestNoAlphaVersionsEnabledByDefault(t *testing.T) {
 	config := DefaultAPIResourceConfigSource()
 	for gv, enable := range config.GroupVersionConfigs {

pkg/master/storageversionhashdata/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["data.go"],
+    importpath = "k8s.io/kubernetes/pkg/master/storageversionhashdata",
+    visibility = ["//visibility:public"],
+    deps = ["//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/master/storageversionhashdata/OWNERS

+approvers:
+- api-approvers
+reviewers:
+- api-reviewers

pkg/master/storageversionhashdata/data.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package storageversionhashdata is for test only.
+package storageversionhashdata
+
+import (
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+// NoStorageVersionHash lists resources that legitimately with empty storage
+// version hash.
+var NoStorageVersionHash = sets.NewString(
+	"v1/bindings",
+	"v1/componentstatuses",
+	"authentication.k8s.io/v1/tokenreviews",
+	"authorization.k8s.io/v1/localsubjectaccessreviews",
+	"authorization.k8s.io/v1/selfsubjectaccessreviews",
+	"authorization.k8s.io/v1/selfsubjectrulesreviews",
+	"authorization.k8s.io/v1/subjectaccessreviews",
+	"authentication.k8s.io/v1beta1/tokenreviews",
+	"authorization.k8s.io/v1beta1/localsubjectaccessreviews",
+	"authorization.k8s.io/v1beta1/selfsubjectaccessreviews",
+	"authorization.k8s.io/v1beta1/selfsubjectrulesreviews",
+	"authorization.k8s.io/v1beta1/subjectaccessreviews",
+	"extensions/v1beta1/replicationcontrollers",
+)
+
+// GVRToStorageVersionHash shouldn't change unless we intentionally change the
+// storage version of a resource.
+var GVRToStorageVersionHash = map[string]string{
+	"v1/configmaps":             "qFsyl6wFWjQ=",
+	"v1/endpoints":              "fWeeMqaN/OA=",
+	"v1/events":                 "r2yiGXH7wu8=",
+	"v1/limitranges":            "EBKMFVe6cwo=",
+	"v1/namespaces":             "Q3oi5N2YM8M=",
+	"v1/nodes":                  "XwShjMxG9Fs=",
+	"v1/persistentvolumeclaims": "QWTyNDq0dC4=",
+	"v1/persistentvolumes":      "HN/zwEC+JgM=",
+	"v1/pods":                   "xPOwRZ+Yhw8=",
+	"v1/podtemplates":           "LIXB2x4IFpk=",
+	"v1/replicationcontrollers": "Jond2If31h0=",
+	"v1/resourcequotas":         "8uhSgffRX6w=",
+	"v1/secrets":                "S6u1pOWzb84=",
+	"v1/serviceaccounts":        "pbx9ZvyFpBE=",
+	"v1/services":               "0/CO1lhkEBI=",
+	"autoscaling/v1/horizontalpodautoscalers":      "oQlkt7f5j/A=",
+	"autoscaling/v2beta1/horizontalpodautoscalers": "oQlkt7f5j/A=",
+	"autoscaling/v2beta2/horizontalpodautoscalers": "oQlkt7f5j/A=",
+	"batch/v1/jobs":          "mudhfqk/qZY=",
+	"batch/v1beta1/cronjobs": "h/JlFAZkyyY=",
+	"certificates.k8s.io/v1beta1/certificatesigningrequests":               "UQh3YTCDIf0=",
+	"coordination.k8s.io/v1beta1/leases":                                   "/sY7hl8ol1U=",
+	"coordination.k8s.io/v1/leases":                                        "/sY7hl8ol1U=",
+	"extensions/v1beta1/daemonsets":                                        "dd7pWHUlMKQ=",
+	"extensions/v1beta1/deployments":                                       "8aSe+NMegvE=",
+	"extensions/v1beta1/ingresses":                                         "Ejja63IbU0E=",
+	"extensions/v1beta1/networkpolicies":                                   "YpfwF18m1G8=",
+	"extensions/v1beta1/podsecuritypolicies":                               "khBLobUXkqA=",
+	"extensions/v1beta1/replicasets":                                       "P1RzHs8/mWQ=",
+	"networking.k8s.io/v1/networkpolicies":                                 "YpfwF18m1G8=",
+	"networking.k8s.io/v1beta1/ingresses":                                  "Ejja63IbU0E=",
+	"node.k8s.io/v1beta1/runtimeclasses":                                   "8nMHWqj34s0=",
+	"policy/v1beta1/poddisruptionbudgets":                                  "6BGBu0kpHtk=",
+	"policy/v1beta1/podsecuritypolicies":                                   "khBLobUXkqA=",
+	"rbac.authorization.k8s.io/v1/clusterrolebindings":                     "48tpQ8gZHFc=",
+	"rbac.authorization.k8s.io/v1/clusterroles":                            "bYE5ZWDrJ44=",
+	"rbac.authorization.k8s.io/v1/rolebindings":                            "eGsCzGH6b1g=",
+	"rbac.authorization.k8s.io/v1/roles":                                   "7FuwZcIIItM=",
+	"rbac.authorization.k8s.io/v1beta1/clusterrolebindings":                "48tpQ8gZHFc=",
+	"rbac.authorization.k8s.io/v1beta1/clusterroles":                       "bYE5ZWDrJ44=",
+	"rbac.authorization.k8s.io/v1beta1/rolebindings":                       "eGsCzGH6b1g=",
+	"rbac.authorization.k8s.io/v1beta1/roles":                              "7FuwZcIIItM=",
+	"scheduling.k8s.io/v1beta1/priorityclasses":                            "D3vHs+OgrtA=",
+	"scheduling.k8s.io/v1/priorityclasses":                                 "D3vHs+OgrtA=",
+	"storage.k8s.io/v1/storageclasses":                                     "K+m6uJwbjGY=",
+	"storage.k8s.io/v1/volumeattachments":                                  "vQAqD28V4AY=",
+	"storage.k8s.io/v1beta1/csidrivers":                                    "hL6j/rwBV5w=",
+	"storage.k8s.io/v1beta1/csinodes":                                      "Pe62DkZtjuo=",
+	"storage.k8s.io/v1beta1/storageclasses":                                "K+m6uJwbjGY=",
+	"storage.k8s.io/v1beta1/volumeattachments":                             "vQAqD28V4AY=",
+	"apps/v1/controllerrevisions":                                          "85nkx63pcBU=",
+	"apps/v1/daemonsets":                                                   "dd7pWHUlMKQ=",
+	"apps/v1/deployments":                                                  "8aSe+NMegvE=",
+	"apps/v1/replicasets":                                                  "P1RzHs8/mWQ=",
+	"apps/v1/statefulsets":                                                 "H+vl74LkKdo=",
+	"apps/v1beta2/controllerrevisions":                                     "85nkx63pcBU=",
+	"apps/v1beta2/daemonsets":                                              "dd7pWHUlMKQ=",
+	"apps/v1beta2/deployments":                                             "8aSe+NMegvE=",
+	"apps/v1beta2/replicasets":                                             "P1RzHs8/mWQ=",
+	"apps/v1beta2/statefulsets":                                            "H+vl74LkKdo=",
+	"apps/v1beta1/controllerrevisions":                                     "85nkx63pcBU=",
+	"apps/v1beta1/deployments":                                             "8aSe+NMegvE=",
+	"apps/v1beta1/statefulsets":                                            "H+vl74LkKdo=",
+	"admissionregistration.k8s.io/v1beta1/mutatingwebhookconfigurations":   "yxW1cpLtfp8=",
+	"admissionregistration.k8s.io/v1beta1/validatingwebhookconfigurations": "P9NhrezfnWE=",
+	"events.k8s.io/v1beta1/events":                                         "r2yiGXH7wu8=",
+}

pkg/registry/core/namespace/storage/storage.go

@@ -226,6 +226,12 @@ func (r *REST) ShortNames() []string {
 	return []string{"ns"}
 }
 
+var _ rest.StorageVersionProvider = &REST{}
+
+func (r *REST) StorageVersion() runtime.GroupVersioner {
+	return r.store.StorageVersion()
+}
+
 func (r *StatusREST) New() runtime.Object {
 	return r.store.New()
 }

pkg/registry/core/service/storage/rest.go

@@ -77,6 +77,7 @@ type ServiceStorage interface {
 	rest.Watcher
 	rest.TableConvertor
 	rest.Exporter
+	rest.StorageVersionProvider
 }
 
 type EndpointsStorage interface {
@@ -108,11 +109,16 @@ func NewREST(
 }
 
 var (
-	_ ServiceStorage          = &REST{}
-	_ rest.CategoriesProvider = &REST{}
-	_ rest.ShortNamesProvider = &REST{}
+	_ ServiceStorage              = &REST{}
+	_ rest.CategoriesProvider     = &REST{}
+	_ rest.ShortNamesProvider     = &REST{}
+	_ rest.StorageVersionProvider = &REST{}
 )
 
+func (rs *REST) StorageVersion() runtime.GroupVersioner {
+	return rs.services.StorageVersion()
+}
+
 // ShortNames implements the ShortNamesProvider interface. Returns a list of short names for a resource.
 func (rs *REST) ShortNames() []string {
 	return []string{"svc"}

pkg/registry/core/service/storage/rest_test.go

@@ -159,6 +159,10 @@ func (s *serviceStorage) Export(ctx context.Context, name string, opts metav1.Ex
 	panic("not implemented")
 }
 
+func (s *serviceStorage) StorageVersion() runtime.GroupVersioner {
+	panic("not implemented")
+}
+
 func generateRandomNodePort() int32 {
 	return int32(rand.IntnRange(30001, 30999))
 }

pkg/registry/rbac/clusterrole/policybased/storage.go

@@ -50,6 +50,16 @@ func (r *Storage) NamespaceScoped() bool {
 	return false
 }
 
+func (r *Storage) StorageVersion() runtime.GroupVersioner {
+	svp, ok := r.StandardStorage.(rest.StorageVersionProvider)
+	if !ok {
+		return nil
+	}
+	return svp.StorageVersion()
+}
+
+var _ rest.StorageVersionProvider = &Storage{}
+
 var fullAuthority = []rbac.PolicyRule{
 	rbac.NewRule("*").Groups("*").Resources("*").RuleOrDie(),
 	rbac.NewRule("*").URLs("*").RuleOrDie(),

pkg/registry/rbac/clusterrolebinding/policybased/storage.go

@@ -51,6 +51,16 @@ func (r *Storage) NamespaceScoped() bool {
 	return false
 }
 
+func (r *Storage) StorageVersion() runtime.GroupVersioner {
+	svp, ok := r.StandardStorage.(rest.StorageVersionProvider)
+	if !ok {
+		return nil
+	}
+	return svp.StorageVersion()
+}
+
+var _ rest.StorageVersionProvider = &Storage{}
+
 func (s *Storage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
 	if rbacregistry.EscalationAllowed(ctx) {
 		return s.StandardStorage.Create(ctx, obj, createValidation, options)

pkg/registry/rbac/role/policybased/storage.go

@@ -49,6 +49,16 @@ func (r *Storage) NamespaceScoped() bool {
 	return true
 }
 
+func (r *Storage) StorageVersion() runtime.GroupVersioner {
+	svp, ok := r.StandardStorage.(rest.StorageVersionProvider)
+	if !ok {
+		return nil
+	}
+	return svp.StorageVersion()
+}
+
+var _ rest.StorageVersionProvider = &Storage{}
+
 func (s *Storage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
 	if rbacregistry.EscalationAllowed(ctx) || rbacregistry.RoleEscalationAuthorized(ctx, s.authorizer) {
 		return s.StandardStorage.Create(ctx, obj, createValidation, options)

pkg/registry/rbac/rolebinding/policybased/storage.go

@@ -52,6 +52,16 @@ func (r *Storage) NamespaceScoped() bool {
 	return true
 }
 
+func (r *Storage) StorageVersion() runtime.GroupVersioner {
+	svp, ok := r.StandardStorage.(rest.StorageVersionProvider)
+	if !ok {
+		return nil
+	}
+	return svp.StorageVersion()
+}
+
+var _ rest.StorageVersionProvider = &Storage{}
+
 func (s *Storage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
 	if rbacregistry.EscalationAllowed(ctx) {
 		return s.StandardStorage.Create(ctx, obj, createValidation, options)

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go

@@ -95,6 +95,7 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 		}
 
 		foundThisVersion := false
+		var storageVersionHash string
 		for _, v := range crd.Spec.Versions {
 			if !v.Served {
 				continue
@@ -113,6 +114,9 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 			if v.Name == version.Version {
 				foundThisVersion = true
 			}
+			if v.Storage {
+				storageVersionHash = discovery.StorageVersionHash(gv.Group, gv.Version, crd.Spec.Names.Kind)
+			}
 		}
 
 		if !foundThisVersion {
@@ -127,13 +131,14 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
 		}
 
 		apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{
-			Name:         crd.Status.AcceptedNames.Plural,
-			SingularName: crd.Status.AcceptedNames.Singular,
-			Namespaced:   crd.Spec.Scope == apiextensions.NamespaceScoped,
-			Kind:         crd.Status.AcceptedNames.Kind,
-			Verbs:        verbs,
-			ShortNames:   crd.Status.AcceptedNames.ShortNames,
-			Categories:   crd.Status.AcceptedNames.Categories,
+			Name:               crd.Status.AcceptedNames.Plural,
+			SingularName:       crd.Status.AcceptedNames.Singular,
+			Namespaced:         crd.Spec.Scope == apiextensions.NamespaceScoped,
+			Kind:               crd.Status.AcceptedNames.Kind,
+			Verbs:              verbs,
+			ShortNames:         crd.Status.AcceptedNames.ShortNames,
+			Categories:         crd.Status.AcceptedNames.Categories,
+			StorageVersionHash: storageVersionHash,
 		})
 
 		subresources, err := apiextensions.GetSubresourcesForVersion(crd, version.Version)

staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD

@@ -32,6 +32,7 @@ go_library(
         "group.go",
         "legacy.go",
         "root.go",
+        "storageversionhash.go",
         "util.go",
         "version.go",
     ],

staging/src/k8s.io/apiserver/pkg/endpoints/discovery/storageversionhash.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package discovery
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+)
+
+// StorageVersionHash calculates the storage version hash for a
+// <group/version/kind> tuple.
+// WARNING: this function is subject to change. Clients shouldn't depend on
+// this function.
+func StorageVersionHash(group, version, kind string) string {
+	gvk := group + "/" + version + "/" + kind
+	if gvk == "" {
+		return ""
+	}
+	bytes := sha256.Sum256([]byte(gvk))
+	// Assuming there are N kinds in the cluster, and the hash is X-byte long,
+	// the chance of colliding hash P(N,X) approximates to 1-e^(-(N^2)/2^(8X+1)).
+	// P(10,000, 8) ~= 2.7*10^(-12), which is low enough.
+	// See https://en.wikipedia.org/wiki/Birthday_problem#Approximations.
+	return base64.StdEncoding.EncodeToString(
+		bytes[:8])
+}

staging/src/k8s.io/apiserver/pkg/endpoints/installer.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
 	"k8s.io/apiserver/pkg/endpoints/handlers"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
@@ -133,6 +134,20 @@ func (a *APIInstaller) newWebService() *restful.WebService {
 	return ws
 }
 
+// calculate the storage gvk, the gvk objects are converted to before persisted to the etcd.
+func getStorageVersionKind(storageVersioner runtime.GroupVersioner, storage rest.Storage, typer runtime.ObjectTyper) (schema.GroupVersionKind, error) {
+	object := storage.New()
+	fqKinds, _, err := typer.ObjectKinds(object)
+	if err != nil {
+		return schema.GroupVersionKind{}, err
+	}
+	gvk, ok := storageVersioner.KindForGroupVersionKinds(fqKinds)
+	if !ok {
+		return schema.GroupVersionKind{}, fmt.Errorf("cannot find the storage version kind for %v", reflect.TypeOf(object))
+	}
+	return gvk, nil
+}
+
 // GetResourceKind returns the external group version kind registered for the given storage
 // object. If the storage object is a subresource and has an override supplied for it, it returns
 // the group version kind supplied in the override.
@@ -227,6 +242,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 	watcher, isWatcher := storage.(rest.Watcher)
 	connecter, isConnecter := storage.(rest.Connecter)
 	storageMeta, isMetadata := storage.(rest.StorageMetadata)
+	storageVersionProvider, isStorageVersionProvider := storage.(rest.StorageVersionProvider)
 	if !isMetadata {
 		storageMeta = defaultStorageMetadata{}
 	}
@@ -365,6 +381,17 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 	tableProvider, _ := storage.(rest.TableConvertor)
 
 	var apiResource metav1.APIResource
+	if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionHash) &&
+		isStorageVersionProvider &&
+		storageVersionProvider.StorageVersion() != nil {
+		versioner := storageVersionProvider.StorageVersion()
+		gvk, err := getStorageVersionKind(versioner, storage, a.group.Typer)
+		if err != nil {
+			return nil, err
+		}
+		apiResource.StorageVersionHash = discovery.StorageVersionHash(gvk.Group, gvk.Version, gvk.Kind)
+	}
+
 	// Get the list of actions for the given scope.
 	switch {
 	case !namespaceScoped:

staging/src/k8s.io/apiserver/pkg/features/kube_features.go

@@ -89,6 +89,13 @@ const (
 	// Server-side apply. Merging happens on the server.
 	ServerSideApply utilfeature.Feature = "ServerSideApply"
 
+	// owner: @caesarxuchao
+	// alpha: v1.14
+	//
+	// Allow apiservers to expose the storage version hash in the discovery
+	// document.
+	StorageVersionHash utilfeature.Feature = "StorageVersionHash"
+
 	// owner: @ksubrmnn
 	// alpha: v1.14
 	//
@@ -118,6 +125,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	APIListChunking:         {Default: true, PreRelease: utilfeature.Beta},
 	DryRun:                  {Default: true, PreRelease: utilfeature.Beta},
 	ServerSideApply:         {Default: false, PreRelease: utilfeature.Alpha},
+	StorageVersionHash:      {Default: false, PreRelease: utilfeature.Alpha},
 	WinOverlay:              {Default: false, PreRelease: utilfeature.Alpha},
 	WinDSR:                  {Default: false, PreRelease: utilfeature.Alpha},
 }

staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go

@@ -177,6 +177,12 @@ type Store struct {
 	// resource. It is wrapped into a "DryRunnableStorage" that will
 	// either pass-through or simply dry-run.
 	Storage DryRunnableStorage
+	// StorageVersioner outputs the <group/version/kind> an object will be
+	// converted to before persisted in etcd, given a list of possible
+	// kinds of the object.
+	// If the StorageVersioner is nil, apiserver will leave the
+	// storageVersionHash as empty in the discovery document.
+	StorageVersioner runtime.GroupVersioner
 	// Called to cleanup clients used by the underlying Storage; optional.
 	DestroyFunc func()
 }
@@ -1287,6 +1293,7 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error {
 			attrFunc,
 			triggerFunc,
 		)
+		e.StorageVersioner = opts.StorageConfig.EncodeVersioner
 
 		if opts.CountMetricPollPeriod > 0 {
 			stopFunc := e.startObservingCount(opts.CountMetricPollPeriod)
@@ -1327,3 +1334,7 @@ func (e *Store) ConvertToTable(ctx context.Context, object runtime.Object, table
 	}
 	return rest.NewDefaultTableConvertor(e.qualifiedResourceFromContext(ctx)).ConvertToTable(ctx, object, tableOptions)
 }
+
+func (e *Store) StorageVersion() runtime.GroupVersioner {
+	return e.StorageVersioner
+}

staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go

@@ -332,3 +332,12 @@ type StorageMetadata interface {
 	// it is not nil. Only the type of the return object matters, the value will be ignored.
 	ProducesObject(verb string) interface{}
 }
+
+// StorageVersionProvider is an optional interface that a storage object can
+// implement if it wishes to disclose its storage version.
+type StorageVersionProvider interface {
+	// StorageVersion returns a group versioner, which will outputs the gvk
+	// an object will be converted to before persisted in etcd, given a
+	// list of kinds the object might belong to.
+	StorageVersion() runtime.GroupVersioner
+}

staging/src/k8s.io/apiserver/pkg/server/storage/storage_codec.go

@@ -40,15 +40,15 @@ type StorageCodecConfig struct {
 
 // NewStorageCodec assembles a storage codec for the provided storage media type, the provided serializer, and the requested
 // storage and memory versions.
-func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, error) {
+func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, runtime.GroupVersioner, error) {
 	mediaType, _, err := mime.ParseMediaType(opts.StorageMediaType)
 	if err != nil {
-		return nil, fmt.Errorf("%q is not a valid mime-type", opts.StorageMediaType)
+		return nil, nil, fmt.Errorf("%q is not a valid mime-type", opts.StorageMediaType)
 	}
 
 	serializer, ok := runtime.SerializerInfoForMediaType(opts.StorageSerializer.SupportedMediaTypes(), mediaType)
 	if !ok {
-		return nil, fmt.Errorf("unable to find serializer for %q", mediaType)
+		return nil, nil, fmt.Errorf("unable to find serializer for %q", mediaType)
 	}
 
 	s := serializer.Serializer
@@ -74,14 +74,16 @@ func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, error) {
 		decoders = opts.DecoderDecoratorFn(decoders)
 	}
 
+	encodeVersioner := runtime.NewMultiGroupVersioner(
+		opts.StorageVersion,
+		schema.GroupKind{Group: opts.StorageVersion.Group},
+		schema.GroupKind{Group: opts.MemoryVersion.Group},
+	)
+
 	// Ensure the storage receives the correct version.
 	encoder = opts.StorageSerializer.EncoderForVersion(
 		encoder,
-		runtime.NewMultiGroupVersioner(
-			opts.StorageVersion,
-			schema.GroupKind{Group: opts.StorageVersion.Group},
-			schema.GroupKind{Group: opts.MemoryVersion.Group},
-		),
+		encodeVersioner,
 	)
 	decoder := opts.StorageSerializer.DecoderToVersion(
 		recognizer.NewDecoder(decoders...),
@@ -92,5 +94,5 @@ func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, error) {
 		),
 	)
 
-	return runtime.NewCodec(encoder, decoder), nil
+	return runtime.NewCodec(encoder, decoder), encodeVersioner, nil
 }

staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go

@@ -86,7 +86,7 @@ type DefaultStorageFactory struct {
 	APIResourceConfigSource APIResourceConfigSource
 
 	// newStorageCodecFn exists to be overwritten for unit testing.
-	newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, err error)
+	newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, encodeVersioner runtime.GroupVersioner, err error)
 }
 
 type groupResourceOverrides struct {
@@ -278,7 +278,7 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*
 	}
 	codecConfig.Config = storageConfig
 
-	storageConfig.Codec, err = s.newStorageCodecFn(codecConfig)
+	storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig)
 	if err != nil {
 		return nil, err
 	}

staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go

@@ -57,6 +57,11 @@ type Config struct {
 	Paging bool
 
 	Codec runtime.Codec
+	// EncodeVersioner is the same groupVersioner used to build the
+	// storage encoder. Given a list of kinds the input object might belong
+	// to, the EncodeVersioner outputs the gvk the object will be
+	// converted to before persisted in etcd.
+	EncodeVersioner runtime.GroupVersioner
 	// Transformer allows the value to be transformed prior to persisting into etcd.
 	Transformer value.Transformer
 

staging/src/k8s.io/sample-apiserver/pkg/cmd/server/BUILD

@@ -11,10 +11,13 @@ go_library(
     importmap = "k8s.io/kubernetes/vendor/k8s.io/sample-apiserver/pkg/cmd/server",
     importpath = "k8s.io/sample-apiserver/pkg/cmd/server",
     deps = [
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/sample-apiserver/pkg/admission/plugin/banflunder:go_default_library",
         "//staging/src/k8s.io/sample-apiserver/pkg/admission/wardleinitializer:go_default_library",
         "//staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1:go_default_library",

staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go

@@ -23,10 +23,13 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apiserver/pkg/admission"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/sample-apiserver/pkg/admission/plugin/banflunder"
 	"k8s.io/sample-apiserver/pkg/admission/wardleinitializer"
 	"k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
@@ -56,7 +59,7 @@ func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions {
 		StdOut: out,
 		StdErr: errOut,
 	}
-
+	o.RecommendedOptions.Etcd.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1alpha1.SchemeGroupVersion, schema.GroupKind{Group: v1alpha1.GroupName})
 	return o
 }
 
@@ -83,6 +86,7 @@ func NewCommandStartWardleServer(defaults *WardleServerOptions, stopCh <-chan st
 
 	flags := cmd.Flags()
 	o.RecommendedOptions.AddFlags(flags)
+	utilfeature.DefaultMutableFeatureGate.AddFlag(flags)
 
 	return cmd
 }

test/e2e/apimachinery/BUILD

@@ -15,6 +15,7 @@ go_library(
         "crd_publish_openapi.go",
         "crd_watch.go",
         "custom_resource_definition.go",
+        "discovery.go",
         "etcd_failure.go",
         "framework.go",
         "garbage_collector.go",
@@ -65,6 +66,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
         "//staging/src/k8s.io/client-go/discovery:go_default_library",

test/e2e/apimachinery/discovery.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apimachinery
+
+import (
+	utilversion "k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var storageVersionServerVersion = utilversion.MustParseSemantic("v1.13.99")
+var _ = SIGDescribe("Discovery", func() {
+	f := framework.NewDefaultFramework("discovery")
+
+	var namespaceName string
+
+	BeforeEach(func() {
+		namespaceName = f.Namespace.Name
+
+		framework.SkipUnlessServerVersionGTE(storageVersionServerVersion, f.ClientSet.Discovery())
+
+		By("Setting up server cert")
+		setupServerCert(namespaceName, serviceName)
+	})
+
+	It("[Feature:StorageVersionHash] Custom resource should have storage version hash", func() {
+		testcrd, err := framework.CreateTestCRD(f)
+		if err != nil {
+			return
+		}
+		defer testcrd.CleanUp()
+		spec := testcrd.Crd.Spec
+		resources, err := testcrd.ApiExtensionClient.Discovery().ServerResourcesForGroupVersion(spec.Group + "/" + spec.Versions[0].Name)
+		if err != nil {
+			framework.Failf("failed to find the discovery doc for %v: %v", resources, err)
+		}
+		found := false
+		var storageVersion string
+		for _, v := range spec.Versions {
+			if v.Storage {
+				storageVersion = v.Name
+			}
+		}
+		// DISCLAIMER: the algorithm of deriving the storageVersionHash
+		// is an implementation detail, which shouldn't be relied on by
+		// the clients. The following calculation is for test purpose
+		// only.
+		expected := discovery.StorageVersionHash(spec.Group, storageVersion, spec.Names.Kind)
+
+		for _, r := range resources.APIResources {
+			if r.Name == spec.Names.Plural {
+				found = true
+				if r.StorageVersionHash != expected {
+					framework.Failf("expected storageVersionHash of %s/%s/%s to be %s, got %s", r.Group, r.Version, r.Name, expected, r.StorageVersionHash)
+				}
+			}
+		}
+		if !found {
+			framework.Failf("didn't find resource %s in the discovery doc", spec.Names.Plural)
+		}
+	})
+})