Merge pull request #20377 from smarterclayton/protobuf_serializer

cmd/libs/go2idl/go-to-protobuf/protobuf/cmd.go

@@ -59,12 +59,16 @@ func New() *Generator {
 		Common:      common,
 		OutputBase:  sourceTree,
 		ProtoImport: []string{defaultProtoImport},
-		Packages: `+k8s.io/kubernetes/pkg/util/intstr,` +
-			`+k8s.io/kubernetes/pkg/api/resource,` +
-			`+k8s.io/kubernetes/pkg/runtime,` +
-			`k8s.io/kubernetes/pkg/api/unversioned,` +
-			`k8s.io/kubernetes/pkg/api/v1,` +
+		Packages: strings.Join([]string{
+			`+k8s.io/kubernetes/pkg/util/intstr`,
+			`+k8s.io/kubernetes/pkg/api/resource`,
+			`+k8s.io/kubernetes/pkg/runtime`,
+			`k8s.io/kubernetes/pkg/api/unversioned`,
+			`k8s.io/kubernetes/pkg/api/v1`,
 			`k8s.io/kubernetes/pkg/apis/extensions/v1beta1`,
+			`k8s.io/kubernetes/pkg/apis/autoscaling/v1`,
+			`k8s.io/kubernetes/pkg/apis/batch/v1`,
+		}, ","),
 		DropEmbeddedFields: "k8s.io/kubernetes/pkg/api/unversioned.TypeMeta",
 	}
 }

pkg/api/serialization_proto_test.go

@@ -25,24 +25,27 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/testapi"
 	apitesting "k8s.io/kubernetes/pkg/api/testing"
+	"k8s.io/kubernetes/pkg/api/unversioned"
 	"k8s.io/kubernetes/pkg/api/v1"
 	_ "k8s.io/kubernetes/pkg/apis/extensions"
 	_ "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
 	"k8s.io/kubernetes/pkg/runtime"
-	"k8s.io/kubernetes/pkg/runtime/protobuf"
+	"k8s.io/kubernetes/pkg/runtime/serializer/protobuf"
 	"k8s.io/kubernetes/pkg/util/diff"
 )
 
 func init() {
-	codecsToTest = append(codecsToTest, func(version string, item runtime.Object) (runtime.Codec, error) {
-		return protobuf.NewCodec(version, api.Scheme, api.Scheme, api.Scheme), nil
+	codecsToTest = append(codecsToTest, func(version unversioned.GroupVersion, item runtime.Object) (runtime.Codec, error) {
+		s := protobuf.NewSerializer(api.Scheme, runtime.ObjectTyperToTyper(api.Scheme))
+		return api.Codecs.CodecForVersions(s, testapi.ExternalGroupVersions(), nil), nil
 	})
 }
 
 func TestProtobufRoundTrip(t *testing.T) {
 	obj := &v1.Pod{}
-	apitesting.FuzzerFor(t, "v1", rand.NewSource(benchmarkSeed)).Fuzz(obj)
+	apitesting.FuzzerFor(t, v1.SchemeGroupVersion, rand.NewSource(benchmarkSeed)).Fuzz(obj)
 	data, err := obj.Marshal()
 	if err != nil {
 		t.Fatal(err)
@@ -57,6 +60,43 @@ func TestProtobufRoundTrip(t *testing.T) {
 	}
 }
 
+// BenchmarkEncodeCodec measures the cost of performing a codec encode, which includes
+// reflection (to clear APIVersion and Kind)
+func BenchmarkEncodeCodecProtobuf(b *testing.B) {
+	items := benchmarkItems()
+	width := len(items)
+	s := protobuf.NewSerializer(nil, nil)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := runtime.Encode(s, &items[i%width]); err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.StopTimer()
+}
+
+// BenchmarkEncodeCodecFromInternalProtobuf measures the cost of performing a codec encode,
+// including conversions and any type setting. This is a "full" encode.
+func BenchmarkEncodeCodecFromInternalProtobuf(b *testing.B) {
+	items := benchmarkItems()
+	width := len(items)
+	encodable := make([]api.Pod, width)
+	for i := range items {
+		if err := api.Scheme.Convert(&items[i], &encodable[i]); err != nil {
+			b.Fatal(err)
+		}
+	}
+	s := protobuf.NewSerializer(nil, nil)
+	codec := api.Codecs.EncoderForVersion(s, v1.SchemeGroupVersion)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := runtime.Encode(codec, &encodable[i%width]); err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.StopTimer()
+}
+
 func BenchmarkEncodeProtobufGeneratedMarshal(b *testing.B) {
 	items := benchmarkItems()
 	width := len(items)

pkg/runtime/protobuf/protobuf.go

-// +build proto
-
-/*
-Copyright 2015 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package protobuf
-
-import (
-	"fmt"
-	"io"
-	"net/url"
-	"reflect"
-
-	"github.com/gogo/protobuf/proto"
-
-	"k8s.io/kubernetes/pkg/api/unversioned"
-	"k8s.io/kubernetes/pkg/runtime"
-)
-
-// NewCodec
-func NewCodec(version string, creater runtime.ObjectCreater, typer runtime.ObjectTyper, convertor runtime.ObjectConvertor) runtime.Codec {
-	return &codec{
-		version:   version,
-		creater:   creater,
-		typer:     typer,
-		convertor: convertor,
-	}
-}
-
-// codec decodes protobuf objects
-type codec struct {
-	version       string
-	outputVersion string
-	creater       runtime.ObjectCreater
-	typer         runtime.ObjectTyper
-	convertor     runtime.ObjectConvertor
-}
-
-var _ runtime.Codec = codec{}
-
-func (c codec) Decode(data []byte) (runtime.Object, error) {
-	unknown := &runtime.Unknown{}
-	if err := proto.Unmarshal(data, unknown); err != nil {
-		return nil, err
-	}
-	obj, err := c.creater.New(unknown.APIVersion, unknown.Kind)
-	if err != nil {
-		return nil, err
-	}
-	pobj, ok := obj.(proto.Message)
-	if !ok {
-		return nil, fmt.Errorf("runtime object is not a proto.Message: %v", reflect.TypeOf(obj))
-	}
-	if unknown.ContentType != runtime.ContentTypeProtobuf {
-		return nil, fmt.Errorf("unmarshal non-protobuf object with protobuf decoder")
-	}
-	if err := proto.Unmarshal(unknown.Raw, pobj); err != nil {
-		return nil, err
-	}
-	if unknown.APIVersion != c.outputVersion {
-		out, err := c.convertor.ConvertToVersion(obj, c.outputVersion)
-		if err != nil {
-			return nil, err
-		}
-		obj = out
-	}
-	return obj, nil
-}
-
-func (c codec) DecodeToVersion(data []byte, version unversioned.GroupVersion) (runtime.Object, error) {
-	return nil, fmt.Errorf("unimplemented")
-}
-
-func (c codec) DecodeInto(data []byte, obj runtime.Object) error {
-	version, kind, err := c.typer.ObjectVersionAndKind(obj)
-	if err != nil {
-		return err
-	}
-	unknown := &runtime.Unknown{}
-	if err := proto.Unmarshal(data, unknown); err != nil {
-		return err
-	}
-	if unknown.ContentType != runtime.ContentTypeProtobuf {
-		return nil, fmt.Errorf("unmarshal non-protobuf object with protobuf decoder")
-	}
-	if unknown.APIVersion == version && unknown.Kind == kind {
-		pobj, ok := obj.(proto.Message)
-		if !ok {
-			return fmt.Errorf("runtime object is not a proto.Message: %v", reflect.TypeOf(obj))
-		}
-
-		return proto.Unmarshal(unknown.Raw, pobj)
-	}
-
-	versioned, err := c.creater.New(unknown.APIVersion, unknown.Kind)
-	if err != nil {
-		return err
-	}
-
-	pobj, ok := versioned.(proto.Message)
-	if !ok {
-		return fmt.Errorf("runtime object is not a proto.Message: %v", reflect.TypeOf(obj))
-	}
-
-	if err := proto.Unmarshal(unknown.Raw, pobj); err != nil {
-		return err
-	}
-	return c.convertor.Convert(versioned, obj)
-}
-
-func (c codec) DecodeIntoWithSpecifiedVersionKind(data []byte, obj runtime.Object, kind unversioned.GroupVersionKind) error {
-	return fmt.Errorf("unimplemented")
-}
-
-func (c codec) DecodeParametersInto(parameters url.Values, obj runtime.Object) error {
-	return fmt.Errorf("unimplemented")
-}
-
-func (c codec) Encode(obj runtime.Object) (data []byte, err error) {
-	version, kind, err := c.typer.ObjectVersionAndKind(obj)
-	if err != nil {
-		return nil, err
-	}
-	if len(version) == 0 {
-		version = c.version
-		converted, err := c.convertor.ConvertToVersion(obj, version)
-		if err != nil {
-			return nil, err
-		}
-		obj = converted
-	}
-	m, ok := obj.(proto.Marshaler)
-	if !ok {
-		return nil, fmt.Errorf("object %v (kind: %s in version: %s) does not implement ProtoBuf marshalling", reflect.TypeOf(obj), kind, c.version)
-	}
-	b, err := m.Marshal()
-	if err != nil {
-		return nil, err
-	}
-	return (&runtime.Unknown{
-		TypeMeta: runtime.TypeMeta{
-			Kind:       kind,
-			APIVersion: version,
-		},
-		Raw:         b,
-		ContentType: runtime.ContentTypeProtobuf,
-	}).Marshal()
-}
-
-func (c codec) EncodeToStream(obj runtime.Object, stream io.Writer) error {
-	return fmt.Errorf("unimplemented")
-}

pkg/runtime/protobuf/doc.go → pkg/runtime/serializer/protobuf/doc.go

@@ -14,5 +14,5 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package protobuf implements ProtoBuf serialization and deserialization.
+// Package protobuf provides a Kubernetes serializer for the protobuf format.
 package protobuf

pkg/runtime/serializer/protobuf/protobuf_test.go

+// +build proto
+
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package protobuf_test
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"testing"
+
+	"k8s.io/kubernetes/pkg/api/unversioned"
+	"k8s.io/kubernetes/pkg/runtime"
+	"k8s.io/kubernetes/pkg/runtime/serializer/protobuf"
+)
+
+type testObject struct {
+	gvk *unversioned.GroupVersionKind
+}
+
+func (d *testObject) GetObjectKind() unversioned.ObjectKind                 { return d }
+func (d *testObject) SetGroupVersionKind(gvk *unversioned.GroupVersionKind) { d.gvk = gvk }
+func (d *testObject) GroupVersionKind() *unversioned.GroupVersionKind       { return d.gvk }
+
+type testMarshalable struct {
+	testObject
+	data []byte
+	err  error
+}
+
+func (d *testMarshalable) Marshal() ([]byte, error) {
+	return d.data, d.err
+}
+
+type testBufferedMarshalable struct {
+	testObject
+	data []byte
+	err  error
+}
+
+func (d *testBufferedMarshalable) Marshal() ([]byte, error) {
+	return nil, fmt.Errorf("not invokable")
+}
+
+func (d *testBufferedMarshalable) MarshalTo(data []byte) (int, error) {
+	copy(data, d.data)
+	return len(d.data), d.err
+}
+
+func (d *testBufferedMarshalable) Size() int {
+	return len(d.data)
+}
+
+func TestRecognize(t *testing.T) {
+	s := protobuf.NewSerializer(nil, nil, "application/protobuf")
+	ignores := [][]byte{
+		nil,
+		{},
+		[]byte("k8s"),
+		{0x6b, 0x38, 0x73, 0x01},
+	}
+	for i, data := range ignores {
+		if ok, err := s.RecognizesData(bytes.NewBuffer(data)); err != nil || ok {
+			t.Errorf("%d: should not recognize data: %v", i, err)
+		}
+	}
+	recognizes := [][]byte{
+		{0x6b, 0x38, 0x73, 0x00},
+		{0x6b, 0x38, 0x73, 0x00, 0x01},
+	}
+	for i, data := range recognizes {
+		if ok, err := s.RecognizesData(bytes.NewBuffer(data)); err != nil || !ok {
+			t.Errorf("%d: should recognize data: %v", i, err)
+		}
+	}
+}
+
+func TestEncode(t *testing.T) {
+	obj1 := &testMarshalable{testObject: testObject{}, data: []byte{}}
+	wire1 := []byte{
+		0x6b, 0x38, 0x73, 0x00, // prefix
+		0x0a, 0x04,
+		0x0a, 0x00, // apiversion
+		0x12, 0x00, // kind
+		0x12, 0x00, // data
+		0x1a, 0x00, // content-type
+		0x22, 0x00, // content-encoding
+	}
+	obj2 := &testMarshalable{
+		testObject: testObject{gvk: &unversioned.GroupVersionKind{Kind: "test", Group: "other", Version: "version"}},
+		data:       []byte{0x01, 0x02, 0x03},
+	}
+	wire2 := []byte{
+		0x6b, 0x38, 0x73, 0x00, // prefix
+		0x0a, 0x15,
+		0x0a, 0x0d, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x2f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, // apiversion
+		0x12, 0x04, 0x74, 0x65, 0x73, 0x74, // kind
+		0x12, 0x03, 0x01, 0x02, 0x03, // data
+		0x1a, 0x00, // content-type
+		0x22, 0x00, // content-encoding
+	}
+
+	err1 := fmt.Errorf("a test error")
+
+	testCases := []struct {
+		obj   runtime.Object
+		data  []byte
+		errFn func(error) bool
+	}{
+		{
+			obj:   &testObject{},
+			errFn: protobuf.IsNotMarshalable,
+		},
+		{
+			obj:  obj1,
+			data: wire1,
+		},
+		{
+			obj:   &testMarshalable{testObject: obj1.testObject, err: err1},
+			errFn: func(err error) bool { return err == err1 },
+		},
+		{
+			// if this test fails, writing the "fast path" marshal is not the same as the "slow path"
+			obj:  &testBufferedMarshalable{testObject: obj1.testObject, data: obj1.data},
+			data: wire1,
+		},
+		{
+			obj:  obj2,
+			data: wire2,
+		},
+		{
+			// if this test fails, writing the "fast path" marshal is not the same as the "slow path"
+			obj:  &testBufferedMarshalable{testObject: obj2.testObject, data: obj2.data},
+			data: wire2,
+		},
+		{
+			obj:   &testBufferedMarshalable{testObject: obj1.testObject, err: err1},
+			errFn: func(err error) bool { return err == err1 },
+		},
+	}
+
+	for i, test := range testCases {
+		s := protobuf.NewSerializer(nil, nil, "application/protobuf")
+		data, err := runtime.Encode(s, test.obj)
+
+		switch {
+		case err == nil && test.errFn != nil:
+			t.Errorf("%d: failed: %v", i, err)
+			continue
+		case err != nil && test.errFn == nil:
+			t.Errorf("%d: failed: %v", i, err)
+			continue
+		case err != nil:
+			if !test.errFn(err) {
+				t.Errorf("%d: failed: %v", i, err)
+			}
+			if data != nil {
+				t.Errorf("%d: should not have returned nil data", i)
+			}
+			continue
+		}
+
+		if test.data != nil && !bytes.Equal(test.data, data) {
+			t.Errorf("%d: unexpected data:\n%s", i, hex.Dump(data))
+			continue
+		}
+
+		if ok, err := s.RecognizesData(bytes.NewBuffer(data)); !ok || err != nil {
+			t.Errorf("%d: did not recognize data generated by call: %v", i, err)
+		}
+	}
+}

pkg/runtime/serializer/protobuf_extension.go

+// +build proto
+
+/*
+Copyright 2014 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package serializer
+
+import (
+	"k8s.io/kubernetes/pkg/runtime"
+	"k8s.io/kubernetes/pkg/runtime/serializer/protobuf"
+)
+
+// contentTypeProtobuf is the protobuf type exposed for Kubernetes. It is private to prevent others from
+// depending on it unintentionally.
+// TODO: potentially move to pkg/api (since it's part of the Kube public API) and pass it in to the
+//   CodecFactory on initialization.
+const contentTypeProtobuf = "application/vnd.kubernetes.protobuf"
+
+func protobufSerializer(scheme *runtime.Scheme) (serializerType, bool) {
+	serializer := protobuf.NewSerializer(scheme, runtime.ObjectTyperToTyper(scheme), contentTypeProtobuf)
+	raw := protobuf.NewRawSerializer(scheme, runtime.ObjectTyperToTyper(scheme), contentTypeProtobuf)
+	return serializerType{
+		AcceptContentTypes: []string{contentTypeProtobuf},
+		ContentType:        contentTypeProtobuf,
+		FileExtensions:     []string{"pb"},
+		Serializer:         serializer,
+		RawSerializer:      raw,
+	}, true
+}
+
+func init() {
+	serializerExtensions = append(serializerExtensions, protobufSerializer)
+}

pkg/runtime/types.go

@@ -38,8 +38,6 @@ type TypeMeta struct {
 
 const (
 	ContentTypeJSON string = "application/json"
-	// TODO: Fix the value.
-	ContentTypeProtobuf string = "application/protobuf"
 )
 
 // RawExtension is used to hold extensions in external versions.

pkg/runtime/types_proto.go

+// +build proto
+
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package runtime
+
+type ProtobufMarshaller interface {
+	MarshalTo(data []byte) (int, error)
+}
+
+// NestedMarshalTo allows a caller to avoid extra allocations during serialization of an Unknown
+// that will contain an object that implements ProtobufMarshaller.
+func (m *Unknown) NestedMarshalTo(data []byte, b ProtobufMarshaller, size uint64) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	data[i] = 0xa
+	i++
+	i = encodeVarintGenerated(data, i, uint64(m.TypeMeta.Size()))
+	n1, err := m.TypeMeta.MarshalTo(data[i:])
+	if err != nil {
+		return 0, err
+	}
+	i += n1
+
+	if b != nil {
+		data[i] = 0x12
+		i++
+		i = encodeVarintGenerated(data, i, size)
+		n2, err := b.MarshalTo(data[i:])
+		if err != nil {
+			return 0, err
+		}
+		i += n2
+	}
+
+	data[i] = 0x1a
+	i++
+	i = encodeVarintGenerated(data, i, uint64(len(m.ContentEncoding)))
+	i += copy(data[i:], m.ContentEncoding)
+
+	data[i] = 0x22
+	i++
+	i = encodeVarintGenerated(data, i, uint64(len(m.ContentType)))
+	i += copy(data[i:], m.ContentType)
+	return i, nil
+}