Merge pull request #75340 from pohly/framework-crd-utils

test/e2e/apimachinery/BUILD

@@ -82,6 +82,7 @@ go_library(
         "//test/e2e/framework:go_default_library",
         "//test/e2e/framework/metrics:go_default_library",
         "//test/utils:go_default_library",
+        "//test/utils/crd:go_default_library",
         "//test/utils/image:go_default_library",
         "//vendor/github.com/go-openapi/spec:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",

test/e2e/apimachinery/crd_conversion_webhook.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/utils/crd"
 	imageutils "k8s.io/kubernetes/test/utils/image"
 
 	. "github.com/onsi/ginkgo"
@@ -100,7 +101,7 @@ var _ = SIGDescribe("CustomResourceConversionWebhook [Feature:CustomResourceWebh
 	})
 
 	It("Should be able to convert from CR v1 to CR v2", func() {
-		testcrd, err := framework.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions,
+		testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions,
 			&v1beta1.WebhookClientConfig{
 				CABundle: context.signingCert,
 				Service: &v1beta1.ServiceReference{
@@ -116,7 +117,7 @@ var _ = SIGDescribe("CustomResourceConversionWebhook [Feature:CustomResourceWebh
 	})
 
 	It("Should be able to convert a non homogeneous list of CRs", func() {
-		testcrd, err := framework.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions,
+		testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions,
 			&v1beta1.WebhookClientConfig{
 				CABundle: context.signingCert,
 				Service: &v1beta1.ServiceReference{
@@ -325,7 +326,7 @@ func testCustomResourceConversionWebhook(f *framework.Framework, crd *v1beta1.Cu
 	verifyV2Object(f, crd, v2crd)
 }
 
-func testCRListConversion(f *framework.Framework, testCrd *framework.TestCrd) {
+func testCRListConversion(f *framework.Framework, testCrd *crd.TestCrd) {
 	crd := testCrd.Crd
 	customResourceClients := testCrd.DynamicClients
 	name1 := "cr-instance-1"
@@ -346,7 +347,7 @@ func testCRListConversion(f *framework.Framework, testCrd *framework.TestCrd) {
 	Expect(err).To(BeNil())
 
 	// Now cr-instance-1 is stored as v1. lets change storage version
-	crd, err = integration.UpdateCustomResourceDefinitionWithRetry(testCrd.ApiExtensionClient, crd.Name, func(c *v1beta1.CustomResourceDefinition) {
+	crd, err = integration.UpdateCustomResourceDefinitionWithRetry(testCrd.APIExtensionClient, crd.Name, func(c *v1beta1.CustomResourceDefinition) {
 		c.Spec.Versions = alternativeApiVersions
 	})
 	Expect(err).To(BeNil())

test/e2e/apimachinery/crd_publish_openapi.go

@@ -37,6 +37,7 @@ import (
 	k8sclientset "k8s.io/client-go/kubernetes"
 	openapiutil "k8s.io/kube-openapi/pkg/util"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/utils/crd"
 	"sigs.k8s.io/yaml"
 )
 
@@ -58,7 +59,7 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish
 			framework.Failf("%v", err)
 		}
 
-		meta := fmt.Sprintf(metaPattern, crd.Kind, crd.ApiGroup, crd.Versions[0].Name, "test-foo")
+		meta := fmt.Sprintf(metaPattern, crd.Kind, crd.APIGroup, crd.Versions[0].Name, "test-foo")
 		ns := fmt.Sprintf("--namespace=%v", f.Namespace.Name)
 
 		By("client-side validation (kubectl create and apply) allows request with known and required properties")
@@ -126,7 +127,7 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish
 			framework.Failf("%v", err)
 		}
 
-		meta := fmt.Sprintf(metaPattern, crd.Kind, crd.ApiGroup, crd.Versions[0].Name, "test-cr")
+		meta := fmt.Sprintf(metaPattern, crd.Kind, crd.APIGroup, crd.Versions[0].Name, "test-cr")
 		ns := fmt.Sprintf("--namespace=%v", f.Namespace.Name)
 
 		By("client-side validation (kubectl create and apply) allows request with any unknown properties")
@@ -164,8 +165,8 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish
 		if err != nil {
 			framework.Failf("%v", err)
 		}
-		if crdFoo.ApiGroup == crdWaldo.ApiGroup {
-			framework.Failf("unexpected: CRDs should be of different group %v, %v", crdFoo.ApiGroup, crdWaldo.ApiGroup)
+		if crdFoo.APIGroup == crdWaldo.APIGroup {
+			framework.Failf("unexpected: CRDs should be of different group %v, %v", crdFoo.APIGroup, crdWaldo.APIGroup)
 		}
 		if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v1beta1"), schemaWaldo); err != nil {
 			framework.Failf("%v", err)
@@ -206,8 +207,8 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish
 		if err != nil {
 			framework.Failf("%v", err)
 		}
-		if crdFoo.ApiGroup != crdWaldo.ApiGroup {
-			framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.ApiGroup, crdWaldo.ApiGroup)
+		if crdFoo.APIGroup != crdWaldo.APIGroup {
+			framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.APIGroup, crdWaldo.APIGroup)
 		}
 		if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v5"), schemaWaldo); err != nil {
 			framework.Failf("%v", err)
@@ -233,8 +234,8 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish
 		if err != nil {
 			framework.Failf("%v", err)
 		}
-		if crdFoo.ApiGroup != crdWaldo.ApiGroup {
-			framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.ApiGroup, crdWaldo.ApiGroup)
+		if crdFoo.APIGroup != crdWaldo.APIGroup {
+			framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.APIGroup, crdWaldo.APIGroup)
 		}
 		if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v6"), schemaWaldo); err != nil {
 			framework.Failf("%v", err)
@@ -265,7 +266,7 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish
 
 		By("rename a version")
 		patch := []byte(`{"spec":{"versions":[{"name":"v2","served":true,"storage":true},{"name":"v4","served":true,"storage":false}]}}`)
-		crdMultiVer.Crd, err = crdMultiVer.ApiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crdMultiVer.GetMetaName(), types.MergePatchType, patch)
+		crdMultiVer.Crd, err = crdMultiVer.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crdMultiVer.GetMetaName(), types.MergePatchType, patch)
 		if err != nil {
 			framework.Failf("%v", err)
 		}
@@ -307,7 +308,7 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish
 
 		By("mark a version not serverd")
 		crd.Crd.Spec.Versions[1].Served = false
-		crd.Crd, err = crd.ApiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd.Crd)
+		crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd.Crd)
 		if err != nil {
 			framework.Failf("%v", err)
 		}
@@ -327,7 +328,7 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish
 	})
 })
 
-func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, versions ...string) (*framework.TestCrd, error) {
+func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, versions ...string) (*crd.TestCrd, error) {
 	group := fmt.Sprintf("%s-test-%s.k8s.io", f.BaseName, groupSuffix)
 	if len(versions) == 0 {
 		return nil, fmt.Errorf("require at least one version for CRD")
@@ -343,7 +344,7 @@ func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, version
 	}
 	apiVersions[0].Storage = true
 
-	crd, err := framework.CreateMultiVersionTestCRD(f, group, apiVersions, nil)
+	crd, err := crd.CreateMultiVersionTestCRD(f, group, apiVersions, nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create CRD: %v", err)
 	}
@@ -366,7 +367,7 @@ func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, version
 	return crd, nil
 }
 
-func cleanupCRD(f *framework.Framework, crd *framework.TestCrd) error {
+func cleanupCRD(f *framework.Framework, crd *crd.TestCrd) error {
 	crd.CleanUp()
 	for _, v := range crd.Versions {
 		name := definitionName(crd, v.Name)
@@ -378,13 +379,13 @@ func cleanupCRD(f *framework.Framework, crd *framework.TestCrd) error {
 }
 
 // patchSchema takes schema in YAML and patches it to given CRD in given version
-func patchSchema(schema []byte, crd *framework.TestCrd) error {
+func patchSchema(schema []byte, crd *crd.TestCrd) error {
 	s, err := utilyaml.ToJSON(schema)
 	if err != nil {
 		return fmt.Errorf("failed to create json patch: %v", err)
 	}
 	patch := []byte(fmt.Sprintf(`{"spec":{"validation":{"openAPIV3Schema":%s}}}`, string(s)))
-	crd.Crd, err = crd.ApiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crd.GetMetaName(), types.MergePatchType, patch)
+	crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(crd.GetMetaName(), types.MergePatchType, patch)
 	return err
 }
 
@@ -484,8 +485,8 @@ func verifyKubectlExplain(name, pattern string) error {
 }
 
 // definitionName returns the openapi definition name for given CRD in given version
-func definitionName(crd *framework.TestCrd, version string) string {
-	return openapiutil.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", crd.ApiGroup, version, crd.Kind))
+func definitionName(crd *crd.TestCrd, version string) string {
+	return openapiutil.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", crd.APIGroup, version, crd.Kind))
 }
 
 var schemaFoo = []byte(`description: Foo CRD for Testing

test/e2e/apimachinery/discovery.go

@@ -20,6 +20,7 @@ import (
 	utilversion "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apiserver/pkg/endpoints/discovery"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/utils/crd"
 
 	. "github.com/onsi/ginkgo"
 )
@@ -40,13 +41,13 @@ var _ = SIGDescribe("Discovery", func() {
 	})
 
 	It("[Feature:StorageVersionHash] Custom resource should have storage version hash", func() {
-		testcrd, err := framework.CreateTestCRD(f)
+		testcrd, err := crd.CreateTestCRD(f)
 		if err != nil {
 			return
 		}
 		defer testcrd.CleanUp()
 		spec := testcrd.Crd.Spec
-		resources, err := testcrd.ApiExtensionClient.Discovery().ServerResourcesForGroupVersion(spec.Group + "/" + spec.Versions[0].Name)
+		resources, err := testcrd.APIExtensionClient.Discovery().ServerResourcesForGroupVersion(spec.Group + "/" + spec.Versions[0].Name)
 		if err != nil {
 			framework.Failf("failed to find the discovery doc for %v: %v", resources, err)
 		}

test/e2e/apimachinery/webhook.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/utils/crd"
 	imageutils "k8s.io/kubernetes/test/utils/image"
 
 	. "github.com/onsi/ginkgo"
@@ -130,7 +131,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
 	})
 
 	It("Should be able to deny custom resource creation", func() {
-		testcrd, err := framework.CreateTestCRD(f)
+		testcrd, err := crd.CreateTestCRD(f)
 		if err != nil {
 			return
 		}
@@ -167,7 +168,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
 	})
 
 	It("Should mutate custom resource", func() {
-		testcrd, err := framework.CreateTestCRD(f)
+		testcrd, err := crd.CreateTestCRD(f)
 		if err != nil {
 			return
 		}
@@ -185,7 +186,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
 	})
 
 	It("Should mutate custom resource with different stored version", func() {
-		testcrd, err := framework.CreateMultiVersionTestCRDWithV1Storage(f)
+		testcrd, err := crd.CreateMultiVersionTestCRDWithV1Storage(f)
 		if err != nil {
 			return
 		}
@@ -1186,7 +1187,7 @@ func cleanWebhookTest(client clientset.Interface, namespaceName string) {
 	_ = client.RbacV1beta1().RoleBindings("kube-system").Delete(roleBindingName, nil)
 }
 
-func registerWebhookForCustomResource(f *framework.Framework, context *certContext, testcrd *framework.TestCrd) func() {
+func registerWebhookForCustomResource(f *framework.Framework, context *certContext, testcrd *crd.TestCrd) func() {
 	client := f.ClientSet
 	By("Registering the custom resource webhook via the AdmissionRegistration API")
 
@@ -1202,7 +1203,7 @@ func registerWebhookForCustomResource(f *framework.Framework, context *certConte
 				Rules: []v1beta1.RuleWithOperations{{
 					Operations: []v1beta1.OperationType{v1beta1.Create, v1beta1.Update},
 					Rule: v1beta1.Rule{
-						APIGroups:   []string{testcrd.ApiGroup},
+						APIGroups:   []string{testcrd.APIGroup},
 						APIVersions: testcrd.GetAPIVersions(),
 						Resources:   []string{testcrd.GetPluralName()},
 					},
@@ -1227,7 +1228,7 @@ func registerWebhookForCustomResource(f *framework.Framework, context *certConte
 	}
 }
 
-func registerMutatingWebhookForCustomResource(f *framework.Framework, context *certContext, testcrd *framework.TestCrd) func() {
+func registerMutatingWebhookForCustomResource(f *framework.Framework, context *certContext, testcrd *crd.TestCrd) func() {
 	client := f.ClientSet
 	By("Registering the mutating webhook for a custom resource via the AdmissionRegistration API")
 
@@ -1243,7 +1244,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c
 				Rules: []v1beta1.RuleWithOperations{{
 					Operations: []v1beta1.OperationType{v1beta1.Create, v1beta1.Update},
 					Rule: v1beta1.Rule{
-						APIGroups:   []string{testcrd.ApiGroup},
+						APIGroups:   []string{testcrd.APIGroup},
 						APIVersions: testcrd.GetAPIVersions(),
 						Resources:   []string{testcrd.GetPluralName()},
 					},
@@ -1262,7 +1263,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c
 				Rules: []v1beta1.RuleWithOperations{{
 					Operations: []v1beta1.OperationType{v1beta1.Create},
 					Rule: v1beta1.Rule{
-						APIGroups:   []string{testcrd.ApiGroup},
+						APIGroups:   []string{testcrd.APIGroup},
 						APIVersions: testcrd.GetAPIVersions(),
 						Resources:   []string{testcrd.GetPluralName()},
 					},
@@ -1338,7 +1339,7 @@ func testMutatingCustomResourceWebhook(f *framework.Framework, crd *apiextension
 	}
 }
 
-func testMultiVersionCustomResourceWebhook(f *framework.Framework, testcrd *framework.TestCrd) {
+func testMultiVersionCustomResourceWebhook(f *framework.Framework, testcrd *crd.TestCrd) {
 	customResourceClient := testcrd.GetV1DynamicClient()
 	By("Creating a custom resource while v1 is storage version")
 	crName := "cr-instance-1"
@@ -1360,7 +1361,7 @@ func testMultiVersionCustomResourceWebhook(f *framework.Framework, testcrd *fram
 
 	By("Patching Custom Resource Definition to set v2 as storage")
 	apiVersionWithV2StoragePatch := fmt.Sprint(`{"spec": {"versions": [{"name": "v1", "storage": false, "served": true},{"name": "v2", "storage": true, "served": true}]}}`)
-	_, err = testcrd.ApiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(testcrd.Crd.Name, types.StrategicMergePatchType, []byte(apiVersionWithV2StoragePatch))
+	_, err = testcrd.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(testcrd.Crd.Name, types.StrategicMergePatchType, []byte(apiVersionWithV2StoragePatch))
 	Expect(err).NotTo(HaveOccurred(), "failed to patch custom resource definition %s in namespace: %s", testcrd.Crd.Name, f.Namespace.Name)
 
 	By("Patching the custom resource while v2 is storage version")
@@ -1427,10 +1428,10 @@ func testCRDDenyWebhook(f *framework.Framework) {
 			Storage: true,
 		},
 	}
-	testcrd := &framework.TestCrd{
+	testcrd := &crd.TestCrd{
 		Name:     name,
 		Kind:     kind,
-		ApiGroup: group,
+		APIGroup: group,
 		Versions: apiVersions,
 	}
 
@@ -1453,7 +1454,7 @@ func testCRDDenyWebhook(f *framework.Framework) {
 			},
 		},
 		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:    testcrd.ApiGroup,
+			Group:    testcrd.APIGroup,
 			Versions: testcrd.Versions,
 			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
 				Plural:   testcrd.GetPluralName(),

test/e2e/framework/BUILD

@@ -7,7 +7,6 @@ go_library(
     srcs = [
         "authorizer_util.go",
         "cleanup.go",
-        "crd_util.go",
         "create.go",
         "deployment_util.go",
         "exec_util.go",
@@ -83,9 +82,6 @@ go_library(
         "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

test/e2e/framework/framework.go

@@ -32,7 +32,6 @@ import (
 	"time"
 
 	"k8s.io/api/core/v1"
-	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -75,7 +74,6 @@ type Framework struct {
 
 	ClientSet                        clientset.Interface
 	KubemarkExternalClusterClientSet clientset.Interface
-	APIExtensionsClientSet           apiextensionsclient.Interface
 
 	InternalClientset *internalclientset.Clientset
 	AggregatorClient  *aggregatorclient.Clientset
@@ -182,8 +180,6 @@ func (f *Framework) BeforeEach() {
 		}
 		f.ClientSet, err = clientset.NewForConfig(config)
 		ExpectNoError(err)
-		f.APIExtensionsClientSet, err = apiextensionsclient.NewForConfig(config)
-		ExpectNoError(err)
 		f.InternalClientset, err = internalclientset.NewForConfig(config)
 		ExpectNoError(err)
 		f.AggregatorClient, err = aggregatorclient.NewForConfig(config)

test/e2e/framework/util.go

@@ -1060,7 +1060,7 @@ func WaitForPersistentVolumeClaimsPhase(phase v1.PersistentVolumeClaimPhase, c c
 func findAvailableNamespaceName(baseName string, c clientset.Interface) (string, error) {
 	var name string
 	err := wait.PollImmediate(Poll, 30*time.Second, func() (bool, error) {
-		name = fmt.Sprintf("%v-%v", baseName, randomSuffix())
+		name = fmt.Sprintf("%v-%v", baseName, RandomSuffix())
 		_, err := c.CoreV1().Namespaces().Get(name, metav1.GetOptions{})
 		if err == nil {
 			// Already taken
@@ -2154,7 +2154,7 @@ func LoadClientset() (*clientset.Clientset, error) {
 //       for pods and replication controllers so we don't
 //       need to use such a function and can instead
 //       use the UUID utility function.
-func randomSuffix() string {
+func RandomSuffix() string {
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	return strconv.Itoa(r.Int() % 10000)
 }

test/e2e/kubectl/BUILD

@@ -34,6 +34,7 @@ go_library(
         "//test/e2e/framework/testfiles:go_default_library",
         "//test/e2e/scheduling:go_default_library",
         "//test/utils:go_default_library",
+        "//test/utils/crd:go_default_library",
         "//test/utils/image:go_default_library",
         "//vendor/github.com/elazarl/goproxy:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",

test/e2e/kubectl/kubectl.go

@@ -61,6 +61,7 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 	"k8s.io/kubernetes/test/e2e/scheduling"
 	testutils "k8s.io/kubernetes/test/utils"
+	"k8s.io/kubernetes/test/utils/crd"
 	uexec "k8s.io/utils/exec"
 
 	"github.com/onsi/ginkgo"
@@ -823,7 +824,7 @@ metadata:
 	framework.KubeDescribe("Kubectl client-side validation", func() {
 		ginkgo.It("should create/apply a CR with unknown fields for CRD with no validation schema", func() {
 			ginkgo.By("create CRD with no validation schema")
-			crd, err := framework.CreateTestCRD(f)
+			crd, err := crd.CreateTestCRD(f)
 			if err != nil {
 				framework.Failf("failed to create test CRD: %v", err)
 			}
@@ -832,7 +833,7 @@ metadata:
 			ginkgo.By("sleep for 10s to wait for potential crd openapi publishing alpha feature")
 			time.Sleep(10 * time.Second)
 
-			meta := fmt.Sprintf(metaPattern, crd.Kind, crd.ApiGroup, crd.Versions[0].Name, "test-cr")
+			meta := fmt.Sprintf(metaPattern, crd.Kind, crd.APIGroup, crd.Versions[0].Name, "test-cr")
 			randomCR := fmt.Sprintf(`{%s,"a":{"b":[{"c":"d"}]}}`, meta)
 			if err := createApplyCustomResource(randomCR, f.Namespace.Name, "test-cr", crd); err != nil {
 				framework.Failf("%v", err)
@@ -841,7 +842,7 @@ metadata:
 
 		ginkgo.It("should create/apply a valid CR for CRD with validation schema", func() {
 			ginkgo.By("prepare CRD with validation schema")
-			crd, err := framework.CreateTestCRD(f)
+			crd, err := crd.CreateTestCRD(f)
 			if err != nil {
 				framework.Failf("failed to create test CRD: %v", err)
 			}
@@ -853,7 +854,7 @@ metadata:
 			ginkgo.By("sleep for 10s to wait for potential crd openapi publishing alpha feature")
 			time.Sleep(10 * time.Second)
 
-			meta := fmt.Sprintf(metaPattern, crd.Kind, crd.ApiGroup, crd.Versions[0].Name, "test-cr")
+			meta := fmt.Sprintf(metaPattern, crd.Kind, crd.APIGroup, crd.Versions[0].Name, "test-cr")
 			validCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"name":"test-bar"}]}}`, meta)
 			if err := createApplyCustomResource(validCR, f.Namespace.Name, "test-cr", crd); err != nil {
 				framework.Failf("%v", err)
@@ -862,7 +863,7 @@ metadata:
 
 		ginkgo.It("should create/apply a valid CR with arbitrary-extra properties for CRD with partially-specified validation schema", func() {
 			ginkgo.By("prepare CRD with partially-specified validation schema")
-			crd, err := framework.CreateTestCRD(f)
+			crd, err := crd.CreateTestCRD(f)
 			if err != nil {
 				framework.Failf("failed to create test CRD: %v", err)
 			}
@@ -874,7 +875,7 @@ metadata:
 			ginkgo.By("sleep for 10s to wait for potential crd openapi publishing alpha feature")
 			time.Sleep(10 * time.Second)
 
-			meta := fmt.Sprintf(metaPattern, crd.Kind, crd.ApiGroup, crd.Versions[0].Name, "test-cr")
+			meta := fmt.Sprintf(metaPattern, crd.Kind, crd.APIGroup, crd.Versions[0].Name, "test-cr")
 			validArbitraryCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"name":"test-bar"}],"extraProperty":"arbitrary-value"}}`, meta)
 			if err := createApplyCustomResource(validArbitraryCR, f.Namespace.Name, "test-cr", crd); err != nil {
 				framework.Failf("%v", err)
@@ -2226,7 +2227,7 @@ func startLocalProxy() (srv *httptest.Server, logs *bytes.Buffer) {
 
 // createApplyCustomResource asserts that given CustomResource be created and applied
 // without being rejected by client-side validation
-func createApplyCustomResource(resource, namespace, name string, crd *framework.TestCrd) error {
+func createApplyCustomResource(resource, namespace, name string, crd *crd.TestCrd) error {
 	ns := fmt.Sprintf("--namespace=%v", namespace)
 	ginkgo.By("successfully create CR")
 	if _, err := framework.RunKubectlInput(resource, ns, "create", "-f", "-"); err != nil {

test/utils/BUILD

@@ -76,6 +76,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//test/utils/crd:all-srcs",
         "//test/utils/harness:all-srcs",
         "//test/utils/image:all-srcs",
         "//test/utils/junit:all-srcs",

test/utils/crd/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["crd_util.go"],
+    importpath = "k8s.io/kubernetes/test/utils/crd",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
+        "//staging/src/k8s.io/client-go/dynamic:go_default_library",
+        "//test/e2e/framework:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/e2e/framework/crd_util.go → test/utils/crd/crd_util.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package framework
+package crd
 
 import (
 	"fmt"
@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/dynamic"
+	"k8s.io/kubernetes/test/e2e/framework"
 )
 
 // CleanCrdFn declares the clean up function needed to remove the CRD
@@ -36,40 +37,40 @@ type CleanCrdFn func() error
 type TestCrd struct {
 	Name               string
 	Kind               string
-	ApiGroup           string
+	APIGroup           string
 	Versions           []apiextensionsv1beta1.CustomResourceDefinitionVersion
-	ApiExtensionClient *crdclientset.Clientset
+	APIExtensionClient *crdclientset.Clientset
 	Crd                *apiextensionsv1beta1.CustomResourceDefinition
 	DynamicClients     map[string]dynamic.ResourceInterface
 	CleanUp            CleanCrdFn
 }
 
-// CreateTestCRD creates a new CRD specifically for the calling test.
-func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiextensionsv1beta1.CustomResourceDefinitionVersion, conversionWebhook *apiextensionsv1beta1.WebhookClientConfig) (*TestCrd, error) {
-	suffix := randomSuffix()
+// CreateMultiVersionTestCRD creates a new CRD specifically for the calling test.
+func CreateMultiVersionTestCRD(f *framework.Framework, group string, apiVersions []apiextensionsv1beta1.CustomResourceDefinitionVersion, conversionWebhook *apiextensionsv1beta1.WebhookClientConfig) (*TestCrd, error) {
+	suffix := framework.RandomSuffix()
 	name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, suffix)
 	kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, suffix)
 	testcrd := &TestCrd{
 		Name:     name,
 		Kind:     kind,
-		ApiGroup: group,
+		APIGroup: group,
 		Versions: apiVersions,
 	}
 
 	// Creating a custom resource definition for use by assorted tests.
-	config, err := LoadConfig()
+	config, err := framework.LoadConfig()
 	if err != nil {
-		Failf("failed to load config: %v", err)
+		framework.Failf("failed to load config: %v", err)
 		return nil, err
 	}
 	apiExtensionClient, err := crdclientset.NewForConfig(config)
 	if err != nil {
-		Failf("failed to initialize apiExtensionClient: %v", err)
+		framework.Failf("failed to initialize apiExtensionClient: %v", err)
 		return nil, err
 	}
 	dynamicClient, err := dynamic.NewForConfig(config)
 	if err != nil {
-		Failf("failed to initialize dynamic client: %v", err)
+		framework.Failf("failed to initialize dynamic client: %v", err)
 		return nil, err
 	}
 
@@ -85,7 +86,7 @@ func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiexte
 	//create CRD and waits for the resource to be recognized and available.
 	crd, err = fixtures.CreateNewCustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient)
 	if err != nil {
-		Failf("failed to create CustomResourceDefinition: %v", err)
+		framework.Failf("failed to create CustomResourceDefinition: %v", err)
 		return nil, err
 	}
 
@@ -97,13 +98,13 @@ func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiexte
 		}
 	}
 
-	testcrd.ApiExtensionClient = apiExtensionClient
+	testcrd.APIExtensionClient = apiExtensionClient
 	testcrd.Crd = crd
 	testcrd.DynamicClients = resourceClients
 	testcrd.CleanUp = func() error {
 		err := fixtures.DeleteCustomResourceDefinition(crd, apiExtensionClient)
 		if err != nil {
-			Failf("failed to delete CustomResourceDefinition(%s): %v", name, err)
+			framework.Failf("failed to delete CustomResourceDefinition(%s): %v", name, err)
 		}
 		return err
 	}
@@ -111,7 +112,7 @@ func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiexte
 }
 
 // CreateTestCRD creates a new CRD specifically for the calling test.
-func CreateTestCRD(f *Framework) (*TestCrd, error) {
+func CreateTestCRD(f *framework.Framework) (*TestCrd, error) {
 	group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
 	apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
 		{
@@ -123,8 +124,8 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) {
 	return CreateMultiVersionTestCRD(f, group, apiVersions, nil)
 }
 
-// CreateTestCRD creates a new CRD specifically for the calling test.
-func CreateMultiVersionTestCRDWithV1Storage(f *Framework) (*TestCrd, error) {
+// CreateMultiVersionTestCRDWithV1Storage creates a new CRD specifically for the calling test.
+func CreateMultiVersionTestCRDWithV1Storage(f *framework.Framework) (*TestCrd, error) {
 	group := fmt.Sprintf("%s-multiversion-crd-test.k8s.io", f.BaseName)
 	apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
 		{
@@ -146,7 +147,7 @@ func newCRDForTest(testcrd *TestCrd) *apiextensionsv1beta1.CustomResourceDefinit
 	return &apiextensionsv1beta1.CustomResourceDefinition{
 		ObjectMeta: metav1.ObjectMeta{Name: testcrd.GetMetaName()},
 		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:    testcrd.ApiGroup,
+			Group:    testcrd.APIGroup,
 			Versions: testcrd.Versions,
 			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
 				Plural:   testcrd.GetPluralName(),
@@ -161,7 +162,7 @@ func newCRDForTest(testcrd *TestCrd) *apiextensionsv1beta1.CustomResourceDefinit
 
 // GetMetaName returns the metaname for the CRD.
 func (c *TestCrd) GetMetaName() string {
-	return c.Name + "s." + c.ApiGroup
+	return c.Name + "s." + c.APIGroup
 }
 
 // GetPluralName returns the plural form of the CRD name
@@ -174,6 +175,7 @@ func (c *TestCrd) GetListName() string {
 	return c.Name + "List"
 }
 
+// GetAPIVersions returns the API versions served by the CRD.
 func (c *TestCrd) GetAPIVersions() []string {
 	ret := []string{}
 	for _, v := range c.Versions {
@@ -184,6 +186,7 @@ func (c *TestCrd) GetAPIVersions() []string {
 	return ret
 }
 
+// GetV1DynamicClient returns the dynamic client for v1.
 func (c *TestCrd) GetV1DynamicClient() dynamic.ResourceInterface {
 	return c.DynamicClients["v1"]
 }
@@ -195,6 +198,6 @@ func (c *TestCrd) PatchSchema(schema []byte) error {
 		return fmt.Errorf("failed to create json patch: %v", err)
 	}
 	patch := []byte(fmt.Sprintf(`{"spec":{"validation":{"openAPIV3Schema":%s}}}`, string(s)))
-	c.Crd, err = c.ApiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(c.GetMetaName(), types.MergePatchType, patch)
+	c.Crd, err = c.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Patch(c.GetMetaName(), types.MergePatchType, patch)
 	return err
 }