Skip to content

K
k3s
Unverified Commit f39158a3 authored by Monis Khan's avatar Monis Khan
Browse files
Options

Add CRDs to etcd storage path test

This change updates the etcd storage path test to exercise custom resource storage by creating custom resource definitions before running the test. Duplicated custom resource definition test logic was consolidated. Signed-off-by: 's avatarMonis Khan <mkhan@redhat.com>
parent 774b1849
Hide whitespace changes
Inline Side-by-side
Showing with 195 additions and 98 deletions
test/integration/auth/BUILD
View file @ f39158a3
......@@ -49,7 +49,6 @@ go_test(
"//staging/src/k8s.io/api/authentication/v1beta1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/storage/v1beta1:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
......@@ -84,6 +83,7 @@ go_test(
"//staging/src/k8s.io/csi-api/pkg/crd:go_default_library",
"//test/e2e/lifecycle/bootstrap:go_default_library",
"//test/integration:go_default_library",
"//test/integration/etcd:go_default_library",
"//test/integration/framework:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/gopkg.in/square/go-jose.v2/jwt:go_default_library",
......
test/integration/auth/node_test.go
View file @ f39158a3
......@@ -18,11 +18,12 @@ package auth
import (
"fmt"
"io/ioutil"
"strings"
"testing"
"time"
storagev1beta1 "k8s.io/api/storage/v1beta1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
......@@ -32,21 +33,18 @@ import (
utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
externalclientset "k8s.io/client-go/kubernetes"
csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
csicrd "k8s.io/csi-api/pkg/crd"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/pkg/apis/coordination"
"k8s.io/kubernetes/pkg/apis/core"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/apis/policy"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/test/integration/etcd"
"k8s.io/kubernetes/test/integration/framework"
"k8s.io/utils/pointer"
"io/ioutil"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
csicrd "k8s.io/csi-api/pkg/crd"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/pkg/apis/core"
"strings"
)
func TestNodeAuthorizer(t *testing.T) {
......@@ -158,13 +156,7 @@ func TestNodeAuthorizer(t *testing.T) {
t.Fatal(err)
}
crd, err := superuserCRDClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(csicrd.CSINodeInfoCRD())
if err != nil {
t.Fatal(err)
}
if err := waitForEstablishedCRD(superuserCRDClient, crd.Name); err != nil {
t.Fatalf("Failed to establish CSINodeInfo CRD: %v", err)
}
etcd.CreateTestCRDs(t, superuserCRDClient, false, csicrd.CSINodeInfoCRD())
getSecret := func(client clientset.Interface) func() error {
return func() error {
......@@ -672,25 +664,3 @@ func expectAllowed(t *testing.T, f func() error) {
t.Errorf("Expected no error, got %v", err)
}
}
func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
if err != nil {
return false, err
}
for _, cond := range crd.Status.Conditions {
switch cond.Type {
case apiextensionsv1beta1.Established:
if cond.Status == apiextensionsv1beta1.ConditionTrue {
return true, err
}
case apiextensionsv1beta1.NamesAccepted:
if cond.Status == apiextensionsv1beta1.ConditionFalse {
fmt.Printf("Name conflict: %v\n", cond.Reason)
}
}
}
return false, nil
})
}
test/integration/etcd/BUILD
View file @ f39158a3
......@@ -53,6 +53,8 @@ go_library(
"//cmd/kube-apiserver/app:go_default_library",
"//cmd/kube-apiserver/app/options:go_default_library",
"//pkg/master:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
......
test/integration/etcd/data.go
View file @ f39158a3
......@@ -16,7 +16,11 @@ limitations under the License.
package etcd
import "k8s.io/apimachinery/pkg/runtime/schema"
import (
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// GetEtcdStorageData returns etcd data for all persisted objects.
// It is exported so that it can be reused across multiple tests.
......@@ -427,6 +431,23 @@ func GetEtcdStorageData() map[schema.GroupVersionResource]StorageData {
Stub: `{"metadata": {"name": "openshiftwebconsoleconfigs.webconsole.operator.openshift.io"},"spec": {"scope": "Cluster","group": "webconsole.operator.openshift.io","version": "v1alpha1","names": {"kind": "OpenShiftWebConsoleConfig","plural": "openshiftwebconsoleconfigs","singular": "openshiftwebconsoleconfig"}}}`,
ExpectedEtcdPath: "/registry/apiextensions.k8s.io/customresourcedefinitions/openshiftwebconsoleconfigs.webconsole.operator.openshift.io",
},
gvr("cr.bar.com", "v1", "foos"): {
Stub: `{"kind": "Foo", "apiVersion": "cr.bar.com/v1", "metadata": {"name": "cr1foo"}, "color": "blue"}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper
ExpectedEtcdPath: "/registry/cr.bar.com/foos/etcdstoragepathtestnamespace/cr1foo",
},
gvr("custom.fancy.com", "v2", "pants"): {
Stub: `{"kind": "Pant", "apiVersion": "custom.fancy.com/v2", "metadata": {"name": "cr2pant"}, "isFancy": true}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper
ExpectedEtcdPath: "/registry/custom.fancy.com/pants/cr2pant",
},
gvr("awesome.bears.com", "v1", "pandas"): {
Stub: `{"kind": "Panda", "apiVersion": "awesome.bears.com/v1", "metadata": {"name": "cr3panda"}, "weight": 100}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper
ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr3panda",
},
gvr("awesome.bears.com", "v3", "pandas"): {
Stub: `{"kind": "Panda", "apiVersion": "awesome.bears.com/v3", "metadata": {"name": "cr4panda"}, "weight": 300}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper
ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr4panda",
ExpectedGVK: gvkP("awesome.bears.com", "v1", "Panda"),
},
// --
}
}
......@@ -446,6 +467,74 @@ type Prerequisite struct {
Stub string
}
// GetCustomResourceDefinitionData returns the resource definitions that back the custom resources
// included in GetEtcdStorageData. They should be created using CreateTestCRDs before running any tests.
func GetCustomResourceDefinitionData() []*apiextensionsv1beta1.CustomResourceDefinition {
return []*apiextensionsv1beta1.CustomResourceDefinition{
// namespaced with legacy version field
{
ObjectMeta: metav1.ObjectMeta{
Name: "foos.cr.bar.com",
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "cr.bar.com",
Version: "v1",
Scope: apiextensionsv1beta1.NamespaceScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Plural: "foos",
Kind: "Foo",
},
},
},
// cluster scoped with legacy version field
{
ObjectMeta: metav1.ObjectMeta{
Name: "pants.custom.fancy.com",
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "custom.fancy.com",
Version: "v2",
Scope: apiextensionsv1beta1.ClusterScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Plural: "pants",
Kind: "Pant",
},
},
},
// cluster scoped with versions field
{
ObjectMeta: metav1.ObjectMeta{
Name: "pandas.awesome.bears.com",
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "awesome.bears.com",
Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{
{
Name: "v1",
Served: true,
Storage: true,
},
{
Name: "v2",
Served: false,
Storage: false,
},
{
Name: "v3",
Served: true,
Storage: false,
},
},
Scope: apiextensionsv1beta1.ClusterScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Plural: "pandas",
Kind: "Panda",
},
},
},
}
}
func gvr(g, v, r string) schema.GroupVersionResource {
return schema.GroupVersionResource{Group: g, Version: v, Resource: r}
}
......
test/integration/etcd/etcd_storage_path_test.go
View file @ f39158a3
......@@ -99,6 +99,10 @@ func TestEtcdStoragePath(t *testing.T) {
if input, err = jsonToMetaObject([]byte(testData.Stub)); err != nil || input.isEmpty() {
t.Fatalf("invalid test data for %s: %v", gvResource, err)
}
// unset type meta fields - we only set these in the CRD test data and it makes
// any CRD test with an expectedGVK override fail the DeepDerivative test
input.Kind = ""
input.APIVersion = ""
}
all := &[]cleanupData{}
......
test/integration/etcd/server.go
View file @ f39158a3
......@@ -30,6 +30,8 @@ import (
"github.com/coreos/etcd/clientv3"
"github.com/coreos/etcd/clientv3/concurrency"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
......@@ -153,6 +155,9 @@ func StartRealMasterOrDie(t *testing.T) *Master {
t.Fatal(err)
}
// create CRDs so we can make sure that custom resources do not get lost
CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(kubeClientConfig), false, GetCustomResourceDefinitionData()...)
// force cached discovery reset
discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery())
restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
......@@ -169,6 +174,9 @@ func StartRealMasterOrDie(t *testing.T) *Master {
}
close(stopCh)
lock.Unlock()
if err := session.Close(); err != nil {
t.Log(err)
}
}
return &Master{
......@@ -281,3 +289,79 @@ func JSONToUnstructured(stub, namespace string, mapping *meta.RESTMapping, dynam
return dynamicClient.Resource(mapping.Resource).Namespace(namespace), &unstructured.Unstructured{Object: typeMetaAdder}, nil
}
// CreateTestCRDs creates the given CRDs, any failure causes the test to Fatal.
// If skipCrdExistsInDiscovery is true, the CRDs are only checked for the Established condition via their Status.
// If skipCrdExistsInDiscovery is false, the CRDs are checked via discovery, see CrdExistsInDiscovery.
func CreateTestCRDs(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crds ...*apiextensionsv1beta1.CustomResourceDefinition) {
for _, crd := range crds {
createTestCRD(t, client, skipCrdExistsInDiscovery, crd)
}
}
func createTestCRD(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crd *apiextensionsv1beta1.CustomResourceDefinition) {
if _, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
t.Fatalf("Failed to create %s CRD; %v", crd.Name, err)
}
if skipCrdExistsInDiscovery {
if err := waitForEstablishedCRD(client, crd.Name); err != nil {
t.Fatalf("Failed to establish %s CRD; %v", crd.Name, err)
}
return
}
if err := wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
return CrdExistsInDiscovery(client, crd), nil
}); err != nil {
t.Fatalf("Failed to see %s in discovery: %v", crd.Name, err)
}
}
func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
if err != nil {
return false, err
}
for _, cond := range crd.Status.Conditions {
switch cond.Type {
case apiextensionsv1beta1.Established:
if cond.Status == apiextensionsv1beta1.ConditionTrue {
return true, nil
}
}
}
return false, nil
})
}
// CrdExistsInDiscovery checks to see if the given CRD exists in discovery at all served versions.
func CrdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) bool {
var versions []string
if len(crd.Spec.Version) != 0 {
versions = append(versions, crd.Spec.Version)
}
for _, v := range crd.Spec.Versions {
if v.Served {
versions = append(versions, v.Name)
}
}
for _, v := range versions {
if !crdVersionExistsInDiscovery(client, crd, v) {
return false
}
}
return true
}
func crdVersionExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition, version string) bool {
resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version)
if err != nil {
return false
}
for _, resource := range resourceList.APIResources {
if resource.Name == crd.Spec.Names.Plural {
return true
}
}
return false
}
test/integration/master/BUILD
View file @ f39158a3
......@@ -62,6 +62,7 @@ go_test(
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library",
"//test/integration:go_default_library",
"//test/integration/etcd:go_default_library",
"//test/integration/framework:go_default_library",
"//test/utils:go_default_library",
"//vendor/github.com/evanphx/json-patch:go_default_library",
......
test/integration/master/crd_test.go
View file @ f39158a3
......@@ -18,7 +18,6 @@ package master
import (
"encoding/json"
"fmt"
"testing"
"time"
......@@ -37,6 +36,7 @@ import (
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/test/integration/etcd"
"k8s.io/kubernetes/test/integration/framework"
)
......@@ -81,12 +81,8 @@ func TestCRDShadowGroup(t *testing.T) {
},
},
}
if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
t.Fatalf("Failed to create networking group CRD: %v", err)
}
if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil {
t.Fatalf("Failed to establish networking group CRD: %v", err)
}
etcd.CreateTestCRDs(t, apiextensionsclient, true, crd)
// wait to give aggregator time to update
time.Sleep(2 * time.Second)
......@@ -97,11 +93,7 @@ func TestCRDShadowGroup(t *testing.T) {
}
t.Logf("Checking that crd resource does not show up in networking group")
found, err := crdExistsInDiscovery(apiextensionsclient, crd)
if err != nil {
t.Fatalf("unexpected discovery error: %v", err)
}
if found {
if etcd.CrdExistsInDiscovery(apiextensionsclient, crd) {
t.Errorf("CRD resource shows up in discovery, but shouldn't.")
}
}
......@@ -137,17 +129,7 @@ func TestCRD(t *testing.T) {
},
},
}
if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
t.Fatalf("Failed to create foos.cr.bar.com CRD; %v", err)
}
if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil {
t.Fatalf("Failed to establish foos.cr.bar.com CRD: %v", err)
}
if err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) {
return crdExistsInDiscovery(apiextensionsclient, crd)
}); err != nil {
t.Fatalf("Failed to see foos.cr.bar.com in discovery: %v", err)
}
etcd.CreateTestCRDs(t, apiextensionsclient, false, crd)
t.Logf("Trying to access foos.cr.bar.com with dynamic client")
dynamicClient, err := dynamic.NewForConfig(result.ClientConfig)
......@@ -306,38 +288,3 @@ func unstructuredFoo(foo *Foo) (*unstructured.Unstructured, error) {
}
return ret, nil
}
func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
if err != nil {
return false, err
}
for _, cond := range crd.Status.Conditions {
switch cond.Type {
case apiextensionsv1beta1.Established:
if cond.Status == apiextensionsv1beta1.ConditionTrue {
return true, err
}
case apiextensionsv1beta1.NamesAccepted:
if cond.Status == apiextensionsv1beta1.ConditionFalse {
fmt.Printf("Name conflict: %v\n", cond.Reason)
}
}
}
return false, nil
})
}
func crdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) (bool, error) {
resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + crd.Spec.Version)
if err != nil {
return false, nil
}
for _, resource := range resourceList.APIResources {
if resource.Name == crd.Spec.Names.Plural {
return true, nil
}
}
return false, nil
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment