Add webhook converter integration test suite

hack/verify-symbols.sh

@@ -30,6 +30,7 @@ BADSYMBOLS=(
   "httptest"
   "testify"
   "testing[.]"
+  "TestOnlySetFatalOnDecodeError"
 )
 
 # b/c hyperkube binds everything simply check that for bad symbols

staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD

@@ -12,6 +12,7 @@ go_test(
         "apply_test.go",
         "basic_test.go",
         "change_test.go",
+        "conversion_test.go",
         "finalization_test.go",
         "objectmeta_test.go",
         "registration_test.go",
@@ -30,7 +31,9 @@ go_test(
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/test/integration/convert:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/test/integration/storage:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
@@ -41,11 +44,13 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/json:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/features:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/storage/etcd3:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",
@@ -69,7 +74,9 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//staging/src/k8s.io/apiextensions-apiserver/test/integration/convert:all-srcs",
         "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:all-srcs",
+        "//staging/src/k8s.io/apiextensions-apiserver/test/integration/storage:all-srcs",
     ],
     tags = ["automanaged"],
 )

staging/src/k8s.io/apiextensions-apiserver/test/integration/convert/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["webhook.go"],
+    importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/test/integration/convert",
+    importpath = "k8s.io/apiextensions-apiserver/test/integration/convert",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

staging/src/k8s.io/apiextensions-apiserver/test/integration/convert/webhook.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package convert
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/uuid"
+)
+
+// WaitReadyFunc calls triggerConversionFn periodically and waits until it detects that the webhook
+// conversion server has handled at least 1 conversion request or the timeout is exceeded, in which
+// case an error is returned.
+type WaitReadyFunc func(timeout time.Duration, triggerConversionFn func() error) error
+
+// StartConversionWebhookServerWithWaitReady starts an http server with the provided handler and returns the WebhookClientConfig
+// needed to configure a CRD to use this conversion webhook as its converter.
+// It also returns a WaitReadyFunc to be called after the CRD is configured to wait until the conversion webhook handler
+// accepts at least one conversion request. If the server fails to start, an error is returned.
+// WaitReady is useful when changing the conversion webhook config of an existing CRD because the update does not take effect immediately.
+func StartConversionWebhookServerWithWaitReady(handler http.Handler) (func(), *apiextensionsv1beta1.WebhookClientConfig, WaitReadyFunc, error) {
+	var once sync.Once
+	handlerReadyC := make(chan struct{})
+	readyNotifyHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		once.Do(func() {
+			close(handlerReadyC)
+		})
+		handler.ServeHTTP(w, r)
+	})
+
+	tearDown, webhookConfig, err := StartConversionWebhookServer(readyNotifyHandler)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("error starting webhook server: %v", err)
+	}
+
+	waitReady := func(timeout time.Duration, triggerConversionFn func() error) error {
+		var err error
+		for {
+			select {
+			case <-handlerReadyC:
+				return nil
+			case <-time.After(timeout):
+				return fmt.Errorf("Timed out waiting for CRD webhook converter update, last trigger conversion error: %v", err)
+			case <-time.After(100 * time.Millisecond):
+				err = triggerConversionFn()
+			}
+		}
+	}
+	return tearDown, webhookConfig, waitReady, err
+}
+
+// StartConversionWebhookServer starts an http server with the provided handler and returns the WebhookClientConfig
+// needed to configure a CRD to use this conversion webhook as its converter.
+func StartConversionWebhookServer(handler http.Handler) (func(), *apiextensionsv1beta1.WebhookClientConfig, error) {
+	// Use a unique path for each webhook server. This ensures that all conversion requests
+	// received by the handler are intended for it; if a WebhookClientConfig other than this one
+	// is applied in the api server, conversion requests will not reach the handler (if they
+	// reach the server they will be returned at 404). This helps prevent tests that require a
+	// specific conversion webhook from accidentally using a different one, which could otherwise
+	// cause a test to flake or pass when it should fail. Since updating the conversion client
+	// config of a custom resource definition does not take effect immediately, this is needed
+	// by the WaitReady returned StartConversionWebhookServerWithWaitReady to detect when a
+	// conversion client config change in the api server has taken effect.
+	path := fmt.Sprintf("/conversionwebhook-%s", uuid.NewUUID())
+	roots := x509.NewCertPool()
+	if !roots.AppendCertsFromPEM(localhostCert) {
+		return nil, nil, fmt.Errorf("Failed to append Cert from PEM")
+	}
+	cert, err := tls.X509KeyPair(localhostCert, localhostKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to build cert with error: %+v", err)
+	}
+	webhookMux := http.NewServeMux()
+	webhookMux.Handle(path, handler)
+	webhookServer := httptest.NewUnstartedServer(webhookMux)
+	webhookServer.TLS = &tls.Config{
+		RootCAs:      roots,
+		Certificates: []tls.Certificate{cert},
+	}
+	webhookServer.StartTLS()
+	endpoint := webhookServer.URL + path
+	webhookConfig := &apiextensionsv1beta1.WebhookClientConfig{
+		CABundle: localhostCert,
+		URL:      &endpoint,
+	}
+	return webhookServer.Close, webhookConfig, nil
+}
+
+// ReviewConverterFunc converts an entire ConversionReview.
+type ReviewConverterFunc func(review apiextensionsv1beta1.ConversionReview) (apiextensionsv1beta1.ConversionReview, error)
+
+// NewReviewWebhookHandler creates a handler that delegates the review conversion to the provided ReviewConverterFunc.
+func NewReviewWebhookHandler(t *testing.T, converterFunc ReviewConverterFunc) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		data, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		if contentType := r.Header.Get("Content-Type"); contentType != "application/json" {
+			t.Errorf("contentType=%s, expect application/json", contentType)
+			return
+		}
+
+		review := apiextensionsv1beta1.ConversionReview{}
+		if err := json.Unmarshal(data, &review); err != nil {
+			t.Errorf("Fail to deserialize object: %s with error: %v", string(data), err)
+			http.Error(w, err.Error(), 400)
+			return
+		}
+
+		review, err = converterFunc(review)
+		if err != nil {
+			t.Errorf("Error converting review: %v", err)
+			http.Error(w, err.Error(), 500)
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(review); err != nil {
+			t.Errorf("Marshal of response failed with error: %v", err)
+		}
+	})
+}
+
+// ObjectConverterFunc converts a single custom resource to the desiredAPIVersion and returns it or returns an error.
+type ObjectConverterFunc func(desiredAPIVersion string, customResource runtime.RawExtension) (runtime.RawExtension, error)
+
+// NewObjectConverterWebhookHandler creates a handler that delegates custom resource conversion to the provided ConverterFunc.
+func NewObjectConverterWebhookHandler(t *testing.T, converterFunc ObjectConverterFunc) http.Handler {
+	return NewReviewWebhookHandler(t, func(review apiextensionsv1beta1.ConversionReview) (apiextensionsv1beta1.ConversionReview, error) {
+		converted := []runtime.RawExtension{}
+		errMsgs := []string{}
+		for _, obj := range review.Request.Objects {
+			convertedObj, err := converterFunc(review.Request.DesiredAPIVersion, obj)
+			if err != nil {
+				errMsgs = append(errMsgs, err.Error())
+			}
+
+			converted = append(converted, convertedObj)
+		}
+
+		review.Response = &apiextensionsv1beta1.ConversionResponse{
+			UID:              review.Request.UID,
+			ConvertedObjects: converted,
+		}
+		if len(errMsgs) == 0 {
+			review.Response.Result = metav1.Status{Status: "Success"}
+		} else {
+			review.Response.Result = metav1.Status{Status: "Failure", Message: strings.Join(errMsgs, ", ")}
+		}
+		return review, nil
+	})
+}
+
+// localhostCert was generated from crypto/tls/generate_cert.go with the following command:
+//     go run generate_cert.go  --rsa-bits 512 --host 127.0.0.1,::1,example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h
+var localhostCert = []byte(`-----BEGIN CERTIFICATE-----
+MIIBjzCCATmgAwIBAgIRAKpi2WmTcFrVjxrl5n5YDUEwDQYJKoZIhvcNAQELBQAw
+EjEQMA4GA1UEChMHQWNtZSBDbzAgFw03MDAxMDEwMDAwMDBaGA8yMDg0MDEyOTE2
+MDAwMFowEjEQMA4GA1UEChMHQWNtZSBDbzBcMA0GCSqGSIb3DQEBAQUAA0sAMEgC
+QQC9fEbRszP3t14Gr4oahV7zFObBI4TfA5i7YnlMXeLinb7MnvT4bkfOJzE6zktn
+59zP7UiHs3l4YOuqrjiwM413AgMBAAGjaDBmMA4GA1UdDwEB/wQEAwICpDATBgNV
+HSUEDDAKBggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MC4GA1UdEQQnMCWCC2V4
+YW1wbGUuY29thwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqGSIb3DQEBCwUA
+A0EAUsVE6KMnza/ZbodLlyeMzdo7EM/5nb5ywyOxgIOCf0OOLHsPS9ueGLQX9HEG
+//yjTXuhNcUugExIjM/AIwAZPQ==
+-----END CERTIFICATE-----`)
+
+// localhostKey is the private key for localhostCert.
+var localhostKey = []byte(`-----BEGIN RSA PRIVATE KEY-----
+MIIBOwIBAAJBAL18RtGzM/e3XgavihqFXvMU5sEjhN8DmLtieUxd4uKdvsye9Phu
+R84nMTrOS2fn3M/tSIezeXhg66quOLAzjXcCAwEAAQJBAKcRxH9wuglYLBdI/0OT
+BLzfWPZCEw1vZmMR2FF1Fm8nkNOVDPleeVGTWoOEcYYlQbpTmkGSxJ6ya+hqRi6x
+goECIQDx3+X49fwpL6B5qpJIJMyZBSCuMhH4B7JevhGGFENi3wIhAMiNJN5Q3UkL
+IuSvv03kaPR5XVQ99/UeEetUgGvBcABpAiBJSBzVITIVCGkGc7d+RCf49KTCIklv
+bGWObufAR8Ni4QIgWpILjW8dkGg8GOUZ0zaNA6Nvt6TIv2UWGJ4v5PoV98kCIQDx
+rIiZs5QbKdycsv9gQJzwQAogC8o04X3Zz3dsoX+h4A==
+-----END RSA PRIVATE KEY-----`)

staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["objectreader.go"],
+    importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/test/integration/storage",
+    importpath = "k8s.io/apiextensions-apiserver/test/integration/storage",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
+        "//vendor/github.com/coreos/etcd/clientv3:go_default_library",
+        "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/objectreader.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"path"
+	"time"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/pkg/transport"
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apiserver/pkg/registry/generic"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+)
+
+// EtcdObjectReader provides direct access to custom resource objects stored in etcd.
+type EtcdObjectReader struct {
+	etcdClient    *clientv3.Client
+	storagePrefix string
+	crd           *apiextensionsv1beta1.CustomResourceDefinition
+}
+
+// NewEtcdObjectReader creates a reader for accessing custom resource objects directly from etcd.
+func NewEtcdObjectReader(etcdClient *clientv3.Client, restOptions *generic.RESTOptions, crd *apiextensionsv1beta1.CustomResourceDefinition) *EtcdObjectReader {
+	return &EtcdObjectReader{etcdClient, restOptions.StorageConfig.Prefix, crd}
+}
+
+// WaitForStorageVersion calls the updateObjFn periodically and waits for the version of the custom resource stored in etcd to be set to the provided version.
+// Typically updateObjFn should perform a noop update to the object so that when stored version of a CRD changes, the object is written at the updated storage version.
+// If the timeout is exceeded a error is returned.
+// This is useful when updating the stored version of an existing CRD because the update does not take effect immediately.
+func (s *EtcdObjectReader) WaitForStorageVersion(version string, ns, name string, timeout time.Duration, updateObjFn func()) error {
+	waitCh := time.After(timeout)
+	for {
+		storage, err := s.GetStoredCustomResource(ns, name)
+		if err != nil {
+			return err
+		}
+		if storage.GetObjectKind().GroupVersionKind().Version == version {
+			return nil
+		}
+		select {
+		case <-waitCh:
+			return fmt.Errorf("timed out after %v waiting for storage version to be %s for object (namespace:%s name:%s)", timeout, version, ns, name)
+		case <-time.After(10 * time.Millisecond):
+			updateObjFn()
+		}
+	}
+}
+
+// GetStoredCustomResource gets the storage representation of a custom resource from etcd.
+func (s *EtcdObjectReader) GetStoredCustomResource(ns, name string) (*unstructured.Unstructured, error) {
+	key := path.Join("/", s.storagePrefix, s.crd.Spec.Group, s.crd.Spec.Names.Plural, ns, name)
+	resp, err := s.etcdClient.KV.Get(context.Background(), key)
+	if err != nil {
+		return nil, fmt.Errorf("error getting storage object %s, %s from etcd at key %s: %v", ns, name, key, err)
+	}
+	if len(resp.Kvs) == 0 {
+		return nil, fmt.Errorf("no storage object found for %s, %s in etcd for key %s", ns, name, key)
+	}
+	raw := resp.Kvs[0].Value
+	u := &unstructured.Unstructured{Object: map[string]interface{}{}}
+	if err := json.Unmarshal(raw, u); err != nil {
+		return nil, fmt.Errorf("error deserializing object %s: %v", string(raw), err)
+	}
+	return u, nil
+}
+
+// GetEtcdClients returns an initialized  clientv3.Client and clientv3.KV.
+func GetEtcdClients(config storagebackend.TransportConfig) (*clientv3.Client, clientv3.KV, error) {
+	tlsInfo := transport.TLSInfo{
+		CertFile: config.CertFile,
+		KeyFile:  config.KeyFile,
+		CAFile:   config.CAFile,
+	}
+
+	tlsConfig, err := tlsInfo.ClientConfig()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cfg := clientv3.Config{
+		Endpoints: config.ServerList,
+		TLS:       tlsConfig,
+	}
+
+	c, err := clientv3.New(cfg)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return c, clientv3.NewKV(c), nil
+}

staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go

@@ -56,9 +56,15 @@ func testingDeferOnDecodeError() {
 
 func init() {
 	// check to see if we are running in a test environment
+	TestOnlySetFatalOnDecodeError(true)
 	fatalOnDecodeError, _ = strconv.ParseBool(os.Getenv("KUBE_PANIC_WATCH_DECODE_ERROR"))
 }
 
+// TestOnlySetFatalOnDecodeError should only be used for cases where decode errors are expected and need to be tested. e.g. conversion webhooks.
+func TestOnlySetFatalOnDecodeError(b bool) {
+	fatalOnDecodeError = b
+}
+
 type watcher struct {
 	client      *clientv3.Client
 	codec       runtime.Codec