Add CRDs to etcd storage path test

test/integration/auth/BUILD

@@ -49,7 +49,6 @@ go_test(
         "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1beta1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
@@ -84,6 +83,7 @@ go_test(
         "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library",
         "//test/e2e/lifecycle/bootstrap:go_default_library",
         "//test/integration:go_default_library",
+        "//test/integration/etcd:go_default_library",
         "//test/integration/framework:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/gopkg.in/square/go-jose.v2/jwt:go_default_library",

test/integration/auth/node_test.go

@@ -18,11 +18,12 @@ package auth
 
 import (
 	"fmt"
+	"io/ioutil"
+	"strings"
 	"testing"
 	"time"
 
 	storagev1beta1 "k8s.io/api/storage/v1beta1"
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -32,21 +33,18 @@ import (
 	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	externalclientset "k8s.io/client-go/kubernetes"
 	csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
+	csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
+	csicrd "k8s.io/csi-api/pkg/crd"
+	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 	"k8s.io/kubernetes/pkg/apis/coordination"
+	"k8s.io/kubernetes/pkg/apis/core"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/policy"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/kubernetes/test/integration/etcd"
 	"k8s.io/kubernetes/test/integration/framework"
 	"k8s.io/utils/pointer"
-
-	"io/ioutil"
-	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
-	csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
-	csicrd "k8s.io/csi-api/pkg/crd"
-	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
-	"k8s.io/kubernetes/pkg/apis/core"
-	"strings"
 )
 
 func TestNodeAuthorizer(t *testing.T) {
@@ -158,13 +156,7 @@ func TestNodeAuthorizer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	crd, err := superuserCRDClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(csicrd.CSINodeInfoCRD())
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := waitForEstablishedCRD(superuserCRDClient, crd.Name); err != nil {
-		t.Fatalf("Failed to establish CSINodeInfo CRD: %v", err)
-	}
+	etcd.CreateTestCRDs(t, superuserCRDClient, false, csicrd.CSINodeInfoCRD())
 
 	getSecret := func(client clientset.Interface) func() error {
 		return func() error {
@@ -672,25 +664,3 @@ func expectAllowed(t *testing.T, f func() error) {
 		t.Errorf("Expected no error, got %v", err)
 	}
 }
-
-func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
-	return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
-		crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
-		if err != nil {
-			return false, err
-		}
-		for _, cond := range crd.Status.Conditions {
-			switch cond.Type {
-			case apiextensionsv1beta1.Established:
-				if cond.Status == apiextensionsv1beta1.ConditionTrue {
-					return true, err
-				}
-			case apiextensionsv1beta1.NamesAccepted:
-				if cond.Status == apiextensionsv1beta1.ConditionFalse {
-					fmt.Printf("Name conflict: %v\n", cond.Reason)
-				}
-			}
-		}
-		return false, nil
-	})
-}

test/integration/etcd/BUILD

@@ -53,6 +53,8 @@ go_library(
         "//cmd/kube-apiserver/app:go_default_library",
         "//cmd/kube-apiserver/app/options:go_default_library",
         "//pkg/master:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",

test/integration/etcd/data.go

@@ -16,7 +16,11 @@ limitations under the License.
 
 package etcd
 
-import "k8s.io/apimachinery/pkg/runtime/schema"
+import (
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
 
 // GetEtcdStorageData returns etcd data for all persisted objects.
 // It is exported so that it can be reused across multiple tests.
@@ -427,6 +431,23 @@ func GetEtcdStorageData() map[schema.GroupVersionResource]StorageData {
 			Stub:             `{"metadata": {"name": "openshiftwebconsoleconfigs.webconsole.operator.openshift.io"},"spec": {"scope": "Cluster","group": "webconsole.operator.openshift.io","version": "v1alpha1","names": {"kind": "OpenShiftWebConsoleConfig","plural": "openshiftwebconsoleconfigs","singular": "openshiftwebconsoleconfig"}}}`,
 			ExpectedEtcdPath: "/registry/apiextensions.k8s.io/customresourcedefinitions/openshiftwebconsoleconfigs.webconsole.operator.openshift.io",
 		},
+		gvr("cr.bar.com", "v1", "foos"): {
+			Stub:             `{"kind": "Foo", "apiVersion": "cr.bar.com/v1", "metadata": {"name": "cr1foo"}, "color": "blue"}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper
+			ExpectedEtcdPath: "/registry/cr.bar.com/foos/etcdstoragepathtestnamespace/cr1foo",
+		},
+		gvr("custom.fancy.com", "v2", "pants"): {
+			Stub:             `{"kind": "Pant", "apiVersion": "custom.fancy.com/v2", "metadata": {"name": "cr2pant"}, "isFancy": true}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper
+			ExpectedEtcdPath: "/registry/custom.fancy.com/pants/cr2pant",
+		},
+		gvr("awesome.bears.com", "v1", "pandas"): {
+			Stub:             `{"kind": "Panda", "apiVersion": "awesome.bears.com/v1", "metadata": {"name": "cr3panda"}, "weight": 100}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper
+			ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr3panda",
+		},
+		gvr("awesome.bears.com", "v3", "pandas"): {
+			Stub:             `{"kind": "Panda", "apiVersion": "awesome.bears.com/v3", "metadata": {"name": "cr4panda"}, "weight": 300}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper
+			ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr4panda",
+			ExpectedGVK:      gvkP("awesome.bears.com", "v1", "Panda"),
+		},
 		// --
 	}
 }
@@ -446,6 +467,74 @@ type Prerequisite struct {
 	Stub    string
 }
 
+// GetCustomResourceDefinitionData returns the resource definitions that back the custom resources
+// included in GetEtcdStorageData.  They should be created using CreateTestCRDs before running any tests.
+func GetCustomResourceDefinitionData() []*apiextensionsv1beta1.CustomResourceDefinition {
+	return []*apiextensionsv1beta1.CustomResourceDefinition{
+		// namespaced with legacy version field
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "foos.cr.bar.com",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   "cr.bar.com",
+				Version: "v1",
+				Scope:   apiextensionsv1beta1.NamespaceScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Plural: "foos",
+					Kind:   "Foo",
+				},
+			},
+		},
+		// cluster scoped with legacy version field
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "pants.custom.fancy.com",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   "custom.fancy.com",
+				Version: "v2",
+				Scope:   apiextensionsv1beta1.ClusterScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Plural: "pants",
+					Kind:   "Pant",
+				},
+			},
+		},
+		// cluster scoped with versions field
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "pandas.awesome.bears.com",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group: "awesome.bears.com",
+				Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+					{
+						Name:    "v1",
+						Served:  true,
+						Storage: true,
+					},
+					{
+						Name:    "v2",
+						Served:  false,
+						Storage: false,
+					},
+					{
+						Name:    "v3",
+						Served:  true,
+						Storage: false,
+					},
+				},
+				Scope: apiextensionsv1beta1.ClusterScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Plural: "pandas",
+					Kind:   "Panda",
+				},
+			},
+		},
+	}
+}
+
 func gvr(g, v, r string) schema.GroupVersionResource {
 	return schema.GroupVersionResource{Group: g, Version: v, Resource: r}
 }

test/integration/etcd/etcd_storage_path_test.go

@@ -99,6 +99,10 @@ func TestEtcdStoragePath(t *testing.T) {
 				if input, err = jsonToMetaObject([]byte(testData.Stub)); err != nil || input.isEmpty() {
 					t.Fatalf("invalid test data for %s: %v", gvResource, err)
 				}
+				// unset type meta fields - we only set these in the CRD test data and it makes
+				// any CRD test with an expectedGVK override fail the DeepDerivative test
+				input.Kind = ""
+				input.APIVersion = ""
 			}
 
 			all := &[]cleanupData{}

test/integration/etcd/server.go

@@ -30,6 +30,8 @@ import (
 	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/clientv3/concurrency"
 
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -153,6 +155,9 @@ func StartRealMasterOrDie(t *testing.T) *Master {
 		t.Fatal(err)
 	}
 
+	// create CRDs so we can make sure that custom resources do not get lost
+	CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(kubeClientConfig), false, GetCustomResourceDefinitionData()...)
+
 	// force cached discovery reset
 	discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery())
 	restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
@@ -169,6 +174,9 @@ func StartRealMasterOrDie(t *testing.T) *Master {
 		}
 		close(stopCh)
 		lock.Unlock()
+		if err := session.Close(); err != nil {
+			t.Log(err)
+		}
 	}
 
 	return &Master{
@@ -281,3 +289,79 @@ func JSONToUnstructured(stub, namespace string, mapping *meta.RESTMapping, dynam
 
 	return dynamicClient.Resource(mapping.Resource).Namespace(namespace), &unstructured.Unstructured{Object: typeMetaAdder}, nil
 }
+
+// CreateTestCRDs creates the given CRDs, any failure causes the test to Fatal.
+// If skipCrdExistsInDiscovery is true, the CRDs are only checked for the Established condition via their Status.
+// If skipCrdExistsInDiscovery is false, the CRDs are checked via discovery, see CrdExistsInDiscovery.
+func CreateTestCRDs(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crds ...*apiextensionsv1beta1.CustomResourceDefinition) {
+	for _, crd := range crds {
+		createTestCRD(t, client, skipCrdExistsInDiscovery, crd)
+	}
+}
+
+func createTestCRD(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crd *apiextensionsv1beta1.CustomResourceDefinition) {
+	if _, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
+		t.Fatalf("Failed to create %s CRD; %v", crd.Name, err)
+	}
+	if skipCrdExistsInDiscovery {
+		if err := waitForEstablishedCRD(client, crd.Name); err != nil {
+			t.Fatalf("Failed to establish %s CRD; %v", crd.Name, err)
+		}
+		return
+	}
+	if err := wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
+		return CrdExistsInDiscovery(client, crd), nil
+	}); err != nil {
+		t.Fatalf("Failed to see %s in discovery: %v", crd.Name, err)
+	}
+}
+
+func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
+	return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
+		crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, cond := range crd.Status.Conditions {
+			switch cond.Type {
+			case apiextensionsv1beta1.Established:
+				if cond.Status == apiextensionsv1beta1.ConditionTrue {
+					return true, nil
+				}
+			}
+		}
+		return false, nil
+	})
+}
+
+// CrdExistsInDiscovery checks to see if the given CRD exists in discovery at all served versions.
+func CrdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) bool {
+	var versions []string
+	if len(crd.Spec.Version) != 0 {
+		versions = append(versions, crd.Spec.Version)
+	}
+	for _, v := range crd.Spec.Versions {
+		if v.Served {
+			versions = append(versions, v.Name)
+		}
+	}
+	for _, v := range versions {
+		if !crdVersionExistsInDiscovery(client, crd, v) {
+			return false
+		}
+	}
+	return true
+}
+
+func crdVersionExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition, version string) bool {
+	resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version)
+	if err != nil {
+		return false
+	}
+	for _, resource := range resourceList.APIResources {
+		if resource.Name == crd.Spec.Names.Plural {
+			return true
+		}
+	}
+	return false
+}

test/integration/master/BUILD

@@ -62,6 +62,7 @@ go_test(
         "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library",
         "//test/integration:go_default_library",
+        "//test/integration/etcd:go_default_library",
         "//test/integration/framework:go_default_library",
         "//test/utils:go_default_library",
         "//vendor/github.com/evanphx/json-patch:go_default_library",

test/integration/master/crd_test.go

@@ -18,7 +18,6 @@ package master
 
 import (
 	"encoding/json"
-	"fmt"
 	"testing"
 	"time"
 
@@ -37,6 +36,7 @@ import (
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
+	"k8s.io/kubernetes/test/integration/etcd"
 	"k8s.io/kubernetes/test/integration/framework"
 )
 
@@ -81,12 +81,8 @@ func TestCRDShadowGroup(t *testing.T) {
 			},
 		},
 	}
-	if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
-		t.Fatalf("Failed to create networking group CRD: %v", err)
-	}
-	if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil {
-		t.Fatalf("Failed to establish networking group CRD: %v", err)
-	}
+	etcd.CreateTestCRDs(t, apiextensionsclient, true, crd)
+
 	// wait to give aggregator time to update
 	time.Sleep(2 * time.Second)
 
@@ -97,11 +93,7 @@ func TestCRDShadowGroup(t *testing.T) {
 	}
 
 	t.Logf("Checking that crd resource does not show up in networking group")
-	found, err := crdExistsInDiscovery(apiextensionsclient, crd)
-	if err != nil {
-		t.Fatalf("unexpected discovery error: %v", err)
-	}
-	if found {
+	if etcd.CrdExistsInDiscovery(apiextensionsclient, crd) {
 		t.Errorf("CRD resource shows up in discovery, but shouldn't.")
 	}
 }
@@ -137,17 +129,7 @@ func TestCRD(t *testing.T) {
 			},
 		},
 	}
-	if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
-		t.Fatalf("Failed to create foos.cr.bar.com CRD; %v", err)
-	}
-	if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil {
-		t.Fatalf("Failed to establish foos.cr.bar.com CRD: %v", err)
-	}
-	if err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) {
-		return crdExistsInDiscovery(apiextensionsclient, crd)
-	}); err != nil {
-		t.Fatalf("Failed to see foos.cr.bar.com in discovery: %v", err)
-	}
+	etcd.CreateTestCRDs(t, apiextensionsclient, false, crd)
 
 	t.Logf("Trying to access foos.cr.bar.com with dynamic client")
 	dynamicClient, err := dynamic.NewForConfig(result.ClientConfig)
@@ -306,38 +288,3 @@ func unstructuredFoo(foo *Foo) (*unstructured.Unstructured, error) {
 	}
 	return ret, nil
 }
-
-func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
-	return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
-		crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
-		if err != nil {
-			return false, err
-		}
-		for _, cond := range crd.Status.Conditions {
-			switch cond.Type {
-			case apiextensionsv1beta1.Established:
-				if cond.Status == apiextensionsv1beta1.ConditionTrue {
-					return true, err
-				}
-			case apiextensionsv1beta1.NamesAccepted:
-				if cond.Status == apiextensionsv1beta1.ConditionFalse {
-					fmt.Printf("Name conflict: %v\n", cond.Reason)
-				}
-			}
-		}
-		return false, nil
-	})
-}
-
-func crdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) (bool, error) {
-	resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + crd.Spec.Version)
-	if err != nil {
-		return false, nil
-	}
-	for _, resource := range resourceList.APIResources {
-		if resource.Name == crd.Spec.Names.Plural {
-			return true, nil
-		}
-	}
-	return false, nil
-}