go-to-protobuf: generate protobuf IDL and marshalers for Go structs

cmd/libs/go2idl/go-to-protobuf/.gitignore

+go-to-protobuf

cmd/libs/go2idl/go-to-protobuf/main.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// go-to-protobuf generates a Protobuf IDL from a Go struct, respecting any
+// existing IDL tags on the Go struct.
+package main
+
+import (
+	"k8s.io/kubernetes/cmd/libs/go2idl/go-to-protobuf/protobuf"
+
+	flag "github.com/spf13/pflag"
+)
+
+var g = protobuf.New()
+
+func init() {
+	g.BindFlags(flag.CommandLine)
+}
+
+func main() {
+	flag.Parse()
+	protobuf.Run(g)
+}

cmd/libs/go2idl/go-to-protobuf/protobuf/cmd.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// go-to-protobuf generates a Protobuf IDL from a Go struct, respecting any
+// existing IDL tags on the Go struct.
+package protobuf
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/kubernetes/cmd/libs/go2idl/args"
+	"k8s.io/kubernetes/cmd/libs/go2idl/generator"
+	"k8s.io/kubernetes/cmd/libs/go2idl/namer"
+	"k8s.io/kubernetes/cmd/libs/go2idl/parser"
+	"k8s.io/kubernetes/cmd/libs/go2idl/types"
+
+	flag "github.com/spf13/pflag"
+)
+
+type Generator struct {
+	Common               args.GeneratorArgs
+	Packages             string
+	OutputBase           string
+	ProtoImport          []string
+	Conditional          string
+	Clean                bool
+	OnlyIDL              bool
+	SkipGeneratedRewrite bool
+	DropEmbeddedFields   string
+}
+
+func New() *Generator {
+	sourceTree := args.DefaultSourceTree()
+	common := args.GeneratorArgs{
+		OutputBase:       sourceTree,
+		GoHeaderFilePath: filepath.Join(sourceTree, "k8s.io/kubernetes/hack/boilerplate/boilerplate.go.txt"),
+	}
+	defaultProtoImport := filepath.Join(sourceTree, "k8s.io", "kubernetes", "Godeps", "_workspace", "src", "github.com", "gogo", "protobuf", "protobuf")
+	return &Generator{
+		Common:      common,
+		OutputBase:  sourceTree,
+		ProtoImport: []string{defaultProtoImport},
+		Packages: `+k8s.io/kubernetes/pkg/util/intstr,` +
+			`+k8s.io/kubernetes/pkg/api/resource,` +
+			`+k8s.io/kubernetes/pkg/runtime,` +
+			`k8s.io/kubernetes/pkg/api/unversioned,` +
+			`k8s.io/kubernetes/pkg/api/v1,` +
+			`k8s.io/kubernetes/pkg/apis/extensions/v1beta1`,
+		DropEmbeddedFields: "k8s.io/kubernetes/pkg/api/unversioned.TypeMeta",
+	}
+}
+
+func (g *Generator) BindFlags(flag *flag.FlagSet) {
+	flag.StringVarP(&g.Common.GoHeaderFilePath, "go-header-file", "h", g.Common.GoHeaderFilePath, "File containing boilerplate header text. The string YEAR will be replaced with the current 4-digit year.")
+	flag.BoolVar(&g.Common.VerifyOnly, "verify-only", g.Common.VerifyOnly, "If true, only verify existing output, do not write anything.")
+	flag.StringVarP(&g.Packages, "packages", "p", g.Packages, "comma-separated list of directories to get input types from. Directories prefixed with '-' are not generated, directories prefixed with '+' only create types with explicit IDL instructions.")
+	flag.StringVarP(&g.OutputBase, "output-base", "o", g.OutputBase, "Output base; defaults to $GOPATH/src/")
+	flag.StringSliceVar(&g.ProtoImport, "proto-import", g.ProtoImport, "The search path for the core protobuf .protos, required, defaults to GODEPS on path.")
+	flag.StringVar(&g.Conditional, "conditional", g.Conditional, "An optional Golang build tag condition to add to the generated Go code")
+	flag.BoolVar(&g.Clean, "clean", g.Clean, "If true, remove all generated files for the specified Packages.")
+	flag.BoolVar(&g.OnlyIDL, "only-idl", g.OnlyIDL, "If true, only generate the IDL for each package.")
+	flag.BoolVar(&g.SkipGeneratedRewrite, "skip-generated-rewrite", g.SkipGeneratedRewrite, "If true, skip fixing up the generated.pb.go file (debugging only).")
+	flag.StringVar(&g.DropEmbeddedFields, "drop-embedded-fields", g.DropEmbeddedFields, "Comma-delimited list of embedded Go types to omit from generated protobufs")
+}
+
+const (
+	typesKindProtobuf = "Protobuf"
+)
+
+func Run(g *Generator) {
+	if g.Common.VerifyOnly {
+		g.OnlyIDL = true
+		g.Clean = false
+	}
+
+	b := parser.New()
+	b.AddBuildTags("proto")
+
+	omitTypes := map[types.Name]struct{}{}
+	for _, t := range strings.Split(g.DropEmbeddedFields, ",") {
+		name := types.Name{}
+		if i := strings.LastIndex(t, "."); i != -1 {
+			name.Package, name.Name = t[:i], t[i+1:]
+		} else {
+			name.Name = t
+		}
+		if len(name.Name) == 0 {
+			log.Fatalf("--drop-embedded-types requires names in the form of [GOPACKAGE.]TYPENAME: %v", t)
+		}
+		omitTypes[name] = struct{}{}
+	}
+
+	boilerplate, err := g.Common.LoadGoBoilerplate()
+	if err != nil {
+		log.Fatalf("Failed loading boilerplate: %v", err)
+	}
+
+	protobufNames := NewProtobufNamer()
+	outputPackages := generator.Packages{}
+	for _, d := range strings.Split(g.Packages, ",") {
+		generateAllTypes, outputPackage := true, true
+		switch {
+		case strings.HasPrefix(d, "+"):
+			d = d[1:]
+			generateAllTypes = false
+		case strings.HasPrefix(d, "-"):
+			d = d[1:]
+			outputPackage = false
+		}
+		name := protoSafePackage(d)
+		parts := strings.SplitN(d, "=", 2)
+		if len(parts) > 1 {
+			d = parts[0]
+			name = parts[1]
+		}
+		p := newProtobufPackage(d, name, generateAllTypes, omitTypes)
+		header := append([]byte{}, boilerplate...)
+		header = append(header, p.HeaderText...)
+		p.HeaderText = header
+		protobufNames.Add(p)
+		if outputPackage {
+			outputPackages = append(outputPackages, p)
+		}
+	}
+
+	if !g.Common.VerifyOnly {
+		for _, p := range outputPackages {
+			if err := p.(*protobufPackage).Clean(g.OutputBase); err != nil {
+				log.Fatalf("Unable to clean package %s: %v", p.Name(), err)
+			}
+		}
+	}
+
+	if g.Clean {
+		return
+	}
+
+	for _, p := range protobufNames.List() {
+		if err := b.AddDir(p.Path()); err != nil {
+			log.Fatalf("Unable to add directory %q: %v", p.Path(), err)
+		}
+	}
+
+	c, err := generator.NewContext(
+		b,
+		namer.NameSystems{
+			"public": namer.NewPublicNamer(3),
+			"proto":  protobufNames,
+		},
+		"public",
+	)
+	c.Verify = g.Common.VerifyOnly
+	c.FileTypes["protoidl"] = protoIDLFileType{}
+
+	if err != nil {
+		log.Fatalf("Failed making a context: %v", err)
+	}
+
+	if err := protobufNames.AssignTypesToPackages(c); err != nil {
+		log.Fatalf("Failed to identify Common types: %v", err)
+	}
+
+	if err := c.ExecutePackages(g.OutputBase, outputPackages); err != nil {
+		log.Fatalf("Failed executing generator: %v", err)
+	}
+
+	if g.OnlyIDL {
+		return
+	}
+
+	if _, err := exec.LookPath("protoc"); err != nil {
+		log.Fatalf("Unable to find 'protoc': %v", err)
+	}
+
+	searchArgs := []string{"-I", ".", "-I", g.OutputBase}
+	if len(g.ProtoImport) != 0 {
+		for _, s := range g.ProtoImport {
+			searchArgs = append(searchArgs, "-I", s)
+		}
+	}
+	args := append(searchArgs, fmt.Sprintf("--gogo_out=%s", g.OutputBase))
+
+	buf := &bytes.Buffer{}
+	if len(g.Conditional) > 0 {
+		fmt.Fprintf(buf, "// +build %s\n\n", g.Conditional)
+	}
+	buf.Write(boilerplate)
+
+	for _, outputPackage := range outputPackages {
+		p := outputPackage.(*protobufPackage)
+		path := filepath.Join(g.OutputBase, p.ImportPath())
+		outputPath := filepath.Join(g.OutputBase, p.OutputPath())
+		cmd := exec.Command("protoc", append(args, path)...)
+		out, err := cmd.CombinedOutput()
+		if len(out) > 0 {
+			log.Printf(string(out))
+		}
+		if err != nil {
+			log.Println(strings.Join(cmd.Args, " "))
+			log.Fatalf("Unable to generate protoc on %s: %v", p.PackageName, err)
+		}
+		if !g.SkipGeneratedRewrite {
+			if err := RewriteGeneratedGogoProtobufFile(outputPath, p.GoPackageName(), p.HasGoType, buf.Bytes()); err != nil {
+				log.Fatalf("Unable to rewrite generated %s: %v", outputPath, err)
+			}
+
+			cmd := exec.Command("goimports", "-w", outputPath)
+			out, err := cmd.CombinedOutput()
+			if len(out) > 0 {
+				log.Printf(string(out))
+			}
+			if err != nil {
+				log.Println(strings.Join(cmd.Args, " "))
+				log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
+			}
+
+			cmd = exec.Command("gofmt", "-s", "-w", outputPath)
+			out, err = cmd.CombinedOutput()
+			if len(out) > 0 {
+				log.Printf(string(out))
+			}
+			if err != nil {
+				log.Println(strings.Join(cmd.Args, " "))
+				log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
+			}
+		}
+	}
+}

cmd/libs/go2idl/go-to-protobuf/protobuf/import_tracker.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package protobuf
+
+import (
+	"fmt"
+	"sort"
+
+	"k8s.io/kubernetes/cmd/libs/go2idl/types"
+)
+
+// ImportTracker handles Protobuf package imports
+//
+// TODO: pay attention to the package name (instead of renaming every package).
+// TODO: Figure out the best way to make names for packages that collide.
+type ImportTracker struct {
+	pathToName map[string]string
+	// forbidden names are in here. (e.g. "go" is a directory in which
+	// there is code, but "go" is not a legal name for a package, so we put
+	// it here to prevent us from naming any package "go")
+	nameToPath map[string]string
+	local      types.Name
+}
+
+func NewImportTracker(local types.Name, types ...*types.Type) *ImportTracker {
+	tracker := &ImportTracker{
+		local:      local,
+		pathToName: map[string]string{},
+		nameToPath: map[string]string{
+		// Add other forbidden keywords that also happen to be
+		// package names here.
+		},
+	}
+	tracker.AddTypes(types...)
+	return tracker
+}
+
+// AddNullable ensures that support for the nullable Gogo-protobuf extension is added.
+func (tracker *ImportTracker) AddNullable() {
+	tracker.AddType(&types.Type{
+		Kind: typesKindProtobuf,
+		Name: types.Name{
+			Name:    "nullable",
+			Package: "gogoproto",
+			Path:    "github.com/gogo/protobuf/gogoproto/gogo.proto",
+		},
+	})
+}
+
+func (tracker *ImportTracker) AddTypes(types ...*types.Type) {
+	for _, t := range types {
+		tracker.AddType(t)
+	}
+}
+func (tracker *ImportTracker) AddType(t *types.Type) {
+	if tracker.local.Package == t.Name.Package {
+		return
+	}
+	// Golang type
+	if t.Kind != typesKindProtobuf {
+		// ignore built in package
+		if t.Kind == types.Builtin {
+			return
+		}
+		if _, ok := tracker.nameToPath[t.Name.Package]; !ok {
+			tracker.nameToPath[t.Name.Package] = ""
+		}
+		return
+	}
+	// ignore default proto package
+	if len(t.Name.Package) == 0 {
+		return
+	}
+	path := t.Name.Path
+	if len(path) == 0 {
+		panic(fmt.Sprintf("imported proto package %s must also have a path", t.Name.Package))
+	}
+	if _, ok := tracker.pathToName[path]; ok {
+		return
+	}
+	tracker.nameToPath[t.Name.Package] = path
+	tracker.pathToName[path] = t.Name.Package
+}
+
+func (tracker *ImportTracker) ImportLines() []string {
+	for k, v := range tracker.nameToPath {
+		if len(v) == 0 {
+			panic(fmt.Sprintf("tracking import Go package %s from %s, but no matching proto path set", k, tracker.local.Package))
+		}
+	}
+	out := []string{}
+	for path := range tracker.pathToName {
+		out = append(out, path)
+	}
+	sort.Sort(sort.StringSlice(out))
+	return out
+}
+
+// LocalNameOf returns the name you would use to refer to the package at the
+// specified path within the body of a file.
+func (tracker *ImportTracker) LocalNameOf(path string) string {
+	return tracker.pathToName[path]
+}

cmd/libs/go2idl/go-to-protobuf/protobuf/namer.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package protobuf
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"k8s.io/kubernetes/cmd/libs/go2idl/generator"
+	"k8s.io/kubernetes/cmd/libs/go2idl/types"
+)
+
+type localNamer struct {
+	localPackage types.Name
+}
+
+func (n localNamer) Name(t *types.Type) string {
+	if t.Kind == types.Map {
+		return fmt.Sprintf("map<%s, %s>", n.Name(t.Key), n.Name(t.Elem))
+	}
+	if len(n.localPackage.Package) != 0 && n.localPackage.Package == t.Name.Package {
+		return t.Name.Name
+	}
+	return t.Name.String()
+}
+
+type protobufNamer struct {
+	packages       []*protobufPackage
+	packagesByPath map[string]*protobufPackage
+}
+
+func NewProtobufNamer() *protobufNamer {
+	return &protobufNamer{
+		packagesByPath: make(map[string]*protobufPackage),
+	}
+}
+
+func (n *protobufNamer) Name(t *types.Type) string {
+	if t.Kind == types.Map {
+		return fmt.Sprintf("map<%s, %s>", n.Name(t.Key), n.Name(t.Elem))
+	}
+	return t.Name.String()
+}
+
+func (n *protobufNamer) List() []generator.Package {
+	packages := make([]generator.Package, 0, len(n.packages))
+	for i := range n.packages {
+		packages = append(packages, n.packages[i])
+	}
+	return packages
+}
+
+func (n *protobufNamer) Add(p *protobufPackage) {
+	n.packagesByPath[p.PackagePath] = p
+	n.packages = append(n.packages, p)
+}
+
+func (n *protobufNamer) GoNameToProtoName(name types.Name) types.Name {
+	if p, ok := n.packagesByPath[name.Package]; ok {
+		return types.Name{
+			Name:    name.Name,
+			Package: p.PackageName,
+			Path:    p.ImportPath(),
+		}
+	}
+	for _, p := range n.packages {
+		if _, ok := p.FilterTypes[name]; ok {
+			return types.Name{
+				Name:    name.Name,
+				Package: p.PackageName,
+				Path:    p.ImportPath(),
+			}
+		}
+	}
+	return types.Name{Name: name.Name}
+}
+
+func (n *protobufNamer) protoNameForTypeName(name types.Name) string {
+	packageName := name.Package
+	if len(name.Package) != 0 {
+		if p, ok := n.packagesByPath[packageName]; ok {
+			packageName = p.Name()
+		} else {
+			packageName = protoSafePackage(packageName)
+		}
+	}
+	if len(name.Name) == 0 {
+		return packageName
+	}
+	if len(packageName) > 0 {
+		return packageName + "." + name.Name
+	}
+	return name.Name
+}
+
+func protoSafePackage(name string) string {
+	return strings.Replace(name, "/", ".", -1)
+}
+
+type typeNameSet map[types.Name]*protobufPackage
+
+// assignGoTypeToProtoPackage looks for Go and Protobuf types that are referenced by a type in
+// a package. It will not recurse into protobuf types.
+func assignGoTypeToProtoPackage(p *protobufPackage, t *types.Type, local, global typeNameSet) {
+	newT, isProto := isFundamentalProtoType(t)
+	if isProto {
+		t = newT
+	}
+	if otherP, ok := global[t.Name]; ok {
+		if _, ok := local[t.Name]; !ok {
+			p.Imports.AddType(&types.Type{
+				Kind: typesKindProtobuf,
+				Name: otherP.ProtoTypeName(),
+			})
+		}
+		return
+	}
+	global[t.Name] = p
+	if _, ok := local[t.Name]; ok {
+		return
+	}
+	// don't recurse into existing proto types
+	if isProto {
+		p.Imports.AddType(t)
+		return
+	}
+
+	local[t.Name] = p
+	for _, m := range t.Members {
+		if isPrivateGoName(m.Name) {
+			continue
+		}
+		field := &protoField{}
+		if err := protobufTagToField(reflect.StructTag(m.Tags).Get("protobuf"), field, m, t, p.ProtoTypeName()); err == nil && field.Type != nil {
+			assignGoTypeToProtoPackage(p, field.Type, local, global)
+			continue
+		}
+		assignGoTypeToProtoPackage(p, m.Type, local, global)
+	}
+	// TODO: should methods be walked?
+	if t.Elem != nil {
+		assignGoTypeToProtoPackage(p, t.Elem, local, global)
+	}
+	if t.Key != nil {
+		assignGoTypeToProtoPackage(p, t.Key, local, global)
+	}
+	if t.Underlying != nil {
+		assignGoTypeToProtoPackage(p, t.Underlying, local, global)
+	}
+}
+
+func (n *protobufNamer) AssignTypesToPackages(c *generator.Context) error {
+	global := make(typeNameSet)
+	for _, p := range n.packages {
+		local := make(typeNameSet)
+		p.Imports = NewImportTracker(p.ProtoTypeName())
+		for _, t := range c.Order {
+			if t.Name.Package != p.PackagePath {
+				continue
+			}
+			assignGoTypeToProtoPackage(p, t, local, global)
+		}
+		p.FilterTypes = make(map[types.Name]struct{})
+		p.LocalNames = make(map[string]struct{})
+		for k, v := range local {
+			if v == p {
+				p.FilterTypes[k] = struct{}{}
+				p.LocalNames[k.Name] = struct{}{}
+			}
+		}
+	}
+	return nil
+}

cmd/libs/go2idl/go-to-protobuf/protobuf/package.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package protobuf
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"k8s.io/kubernetes/cmd/libs/go2idl/generator"
+	"k8s.io/kubernetes/cmd/libs/go2idl/types"
+)
+
+func newProtobufPackage(packagePath, packageName string, generateAll bool, omitFieldTypes map[types.Name]struct{}) *protobufPackage {
+	pkg := &protobufPackage{
+		// The protobuf package name (foo.bar.baz)
+		PackageName: packageName,
+		// A path segment relative to the GOPATH root (foo/bar/baz)
+		PackagePath:    packagePath,
+		GenerateAll:    generateAll,
+		OmitFieldTypes: omitFieldTypes,
+		HeaderText: []byte(
+			`
+// This file was autogenerated by the command:
+// $ ` + os.Args[0] + `
+// Do not edit it manually!
+
+`),
+		PackageDocumentation: []byte(fmt.Sprintf(
+			`// Package %s is an autogenerated protobuf IDL.
+`, packageName)),
+	}
+	return pkg
+}
+
+// protobufPackage contains the protobuf implentation of Package.
+type protobufPackage struct {
+	// Short name of package, used in the "package xxxx" line.
+	PackageName string
+	// Import path of the package, and the location on disk of the package.
+	PackagePath string
+	// If true, generate protobuf serializations for all public types.
+	// If false, only generate protobuf serializations for structs that
+	// request serialization.
+	GenerateAll bool
+
+	// Emitted at the top of every file.
+	HeaderText []byte
+
+	// Emitted only for a "doc.go" file; appended to the HeaderText for
+	// that file.
+	PackageDocumentation []byte
+
+	// A list of types to filter to; if not specified all types will be included.
+	FilterTypes map[types.Name]struct{}
+
+	// A list of field types that will be excluded from the output struct
+	OmitFieldTypes map[types.Name]struct{}
+
+	// A list of names that this package exports
+	LocalNames map[string]struct{}
+
+	// An import tracker for this package
+	Imports *ImportTracker
+}
+
+func (p *protobufPackage) Clean(outputBase string) error {
+	for _, s := range []string{p.ImportPath(), p.OutputPath()} {
+		if err := os.Remove(filepath.Join(outputBase, s)); err != nil && !os.IsNotExist(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *protobufPackage) ProtoTypeName() types.Name {
+	return types.Name{
+		Name:    p.Path(),       // the go path "foo/bar/baz"
+		Package: p.Name(),       // the protobuf package "foo.bar.baz"
+		Path:    p.ImportPath(), // the path of the import to get the proto
+	}
+}
+
+func (p *protobufPackage) Name() string { return p.PackageName }
+func (p *protobufPackage) Path() string { return p.PackagePath }
+
+func (p *protobufPackage) Filter(c *generator.Context, t *types.Type) bool {
+	switch t.Kind {
+	case types.Func, types.Chan:
+		return false
+	case types.Struct:
+		if t.Name.Name == "struct{}" {
+			return false
+		}
+	case types.Builtin:
+		return false
+	case types.Alias:
+		return false
+	case types.Slice, types.Array, types.Map:
+		return false
+	case types.Pointer:
+		return false
+	}
+	if _, ok := isFundamentalProtoType(t); ok {
+		return false
+	}
+	_, ok := p.FilterTypes[t.Name]
+	return ok
+}
+
+func (p *protobufPackage) HasGoType(name string) bool {
+	_, ok := p.LocalNames[name]
+	return ok
+}
+
+func (p *protobufPackage) Generators(c *generator.Context) []generator.Generator {
+	generators := []generator.Generator{}
+
+	p.Imports.AddNullable()
+
+	generators = append(generators, &genProtoIDL{
+		DefaultGen: generator.DefaultGen{
+			OptionalName: "generated",
+		},
+		localPackage:   types.Name{Package: p.PackageName, Path: p.PackagePath},
+		localGoPackage: types.Name{Package: p.PackagePath, Name: p.GoPackageName()},
+		imports:        p.Imports,
+		generateAll:    p.GenerateAll,
+		omitFieldTypes: p.OmitFieldTypes,
+	})
+	return generators
+}
+
+func (p *protobufPackage) Header(filename string) []byte {
+	if filename == "doc.go" {
+		return append(p.HeaderText, p.PackageDocumentation...)
+	}
+	return p.HeaderText
+}
+
+func (p *protobufPackage) GoPackageName() string {
+	return filepath.Base(p.PackagePath)
+}
+
+func (p *protobufPackage) ImportPath() string {
+	return filepath.Join(p.PackagePath, "generated.proto")
+}
+
+func (p *protobufPackage) OutputPath() string {
+	return filepath.Join(p.PackagePath, "generated.pb.go")
+}
+
+var (
+	_ = generator.Package(&protobufPackage{})
+)

cmd/libs/go2idl/go-to-protobuf/protobuf/parser.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package protobuf
+
+import (
+	"bytes"
+	"go/format"
+	"io/ioutil"
+	"os"
+
+	"k8s.io/kubernetes/third_party/golang/go/ast"
+	"k8s.io/kubernetes/third_party/golang/go/parser"
+	"k8s.io/kubernetes/third_party/golang/go/printer"
+	"k8s.io/kubernetes/third_party/golang/go/token"
+)
+
+func RewriteGeneratedGogoProtobufFile(name string, packageName string, typeExistsFn func(string) bool, header []byte) error {
+	fset := token.NewFileSet()
+	src, err := ioutil.ReadFile(name)
+	if err != nil {
+		return err
+	}
+	file, err := parser.ParseFile(fset, name, src, parser.DeclarationErrors|parser.ParseComments)
+	if err != nil {
+		return err
+	}
+	cmap := ast.NewCommentMap(fset, file, file.Comments)
+
+	// remove types that are already declared
+	decls := []ast.Decl{}
+	for _, d := range file.Decls {
+		if !dropExistingTypeDeclarations(d, typeExistsFn) {
+			decls = append(decls, d)
+		}
+	}
+	file.Decls = decls
+
+	// remove unmapped comments
+	file.Comments = cmap.Filter(file).Comments()
+
+	b := &bytes.Buffer{}
+	b.Write(header)
+	if err := printer.Fprint(b, fset, file); err != nil {
+		return err
+	}
+
+	body, err := format.Source(b.Bytes())
+	if err != nil {
+		return err
+	}
+
+	f, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if _, err := f.Write(body); err != nil {
+		return err
+	}
+	return f.Close()
+}
+
+func dropExistingTypeDeclarations(decl ast.Decl, existsFn func(string) bool) bool {
+	switch t := decl.(type) {
+	case *ast.GenDecl:
+		if t.Tok != token.TYPE {
+			return false
+		}
+		specs := []ast.Spec{}
+		for _, s := range t.Specs {
+			switch spec := s.(type) {
+			case *ast.TypeSpec:
+				if existsFn(spec.Name.Name) {
+					continue
+				}
+				specs = append(specs, spec)
+			}
+		}
+		if len(specs) == 0 {
+			return true
+		}
+		t.Specs = specs
+	}
+	return false
+}

cmd/libs/go2idl/go-to-protobuf/protoc-gen-gogo/main.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package main defines the protoc-gen-gogo binary we use to generate our proto go files,
+// as well as takes dependencies on the correct gogo/protobuf packages for godeps.
+package main
+
+import (
+	"github.com/gogo/protobuf/vanity/command"
+
+	// dependencies that are required for our packages
+	_ "github.com/gogo/protobuf/gogoproto"
+	_ "github.com/gogo/protobuf/proto"
+	_ "github.com/gogo/protobuf/sortkeys"
+)
+
+func main() {
+	command.Write(command.Generate(command.Read()))
+}

hack/after-build/update-generated-protobuf.sh

+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+# TODO: preinstall protobuf
+if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3.0."* ]]; then
+  echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and"
+  echo "install the platform appropriate Protobuf package for your OS: "
+  echo
+  echo "  https://github.com/google/protobuf/releases"
+  echo
+  echo "WARNING: Protobuf changes are not being validated"
+  # TODO: make error when protobuf is installed
+  exit 0
+fi
+
+gotoprotobuf=$(kube::util::find-binary "go-to-protobuf")
+
+# requires the 'proto' tag to build (will remove when ready)
+# searches for the protoc-gen-gogo extension in the output directory
+# satisfies import of github.com/gogo/protobuf/gogoproto/gogo.proto and the core Google protobuf types
+PATH="${KUBE_ROOT}/_output/local/go/bin:${PATH}" "${gotoprotobuf}" \
+  --conditional="proto" \
+  --proto-import="${KUBE_ROOT}/Godeps/_workspace/src" \
+  --proto-import="${KUBE_ROOT}/third_party/protobuf"

hack/after-build/verify-generated-protobuf.sh

+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+# TODO: preinstall protobuf
+if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3.0."* ]]; then
+  echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and"
+  echo "install the platform appropriate Protobuf package for your OS: "
+  echo
+  echo "  https://github.com/google/protobuf/releases"
+  echo
+  echo "WARNING: Protobuf changes are not being validated"
+  # TODO: make error when protobuf is installed
+  exit 0
+fi
+
+APIROOTS=${APIROOTS:-pkg/api pkg/apis/extensions pkg/apis/metrics pkg/runtime pkg/util/intstr}
+_tmp="${KUBE_ROOT}/_tmp"
+
+cleanup() {
+  rm -rf "${_tmp}"
+}
+
+trap "cleanup" EXIT SIGINT
+
+for APIROOT in ${APIROOTS}; do
+  mkdir -p "${_tmp}/${APIROOT%/*}"
+  cp -a "${KUBE_ROOT}/${APIROOT}" "${_tmp}/${APIROOT}"
+done
+
+"${KUBE_ROOT}/hack/after-build/update-generated-protobuf.sh"
+for APIROOT in ${APIROOTS}; do
+  TMP_APIROOT="${_tmp}/${APIROOT}"
+  echo "diffing ${APIROOT} against freshly generated protobuf"
+  ret=0
+  diff -Naupr -I 'Auto generated by' "${KUBE_ROOT}/${APIROOT}" "${TMP_APIROOT}" || ret=$?
+  cp -a "${TMP_APIROOT}" "${KUBE_ROOT}/${APIROOT%/*}"
+  if [[ $ret -eq 0 ]]; then
+    echo "${APIROOT} up to date."
+  else
+    echo "${APIROOT} is out of date. Please run hack/update-generated-protobuf.sh"
+    exit 1
+  fi
+done

hack/update-generated-protobuf.sh

+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+kube::golang::setup_env
+
+"${KUBE_ROOT}/hack/build-go.sh" cmd/libs/go2idl/go-to-protobuf cmd/libs/go2idl/go-to-protobuf/protoc-gen-gogo
+
+"${KUBE_ROOT}/hack/after-build/update-generated-protobuf.sh" "$@"
+
+# ex: ts=2 sw=2 et filetype=sh

hack/verify-all.sh

@@ -59,7 +59,8 @@ if $SILENT ; then
   echo "Running in the silent mode, run with -v if you want to see script logs."
 fi
 
-EXCLUDE="verify-godeps.sh"
+# remove protobuf until it is part of direct generation
+EXCLUDE="verify-godeps.sh verify-generated-protobuf.sh"
 
 ret=0
 for t in `ls $KUBE_ROOT/hack/verify-*.sh`

hack/verify-generated-protobuf.sh

+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+kube::golang::setup_env
+
+"${KUBE_ROOT}/hack/build-go.sh" cmd/libs/go2idl/go-to-protobuf cmd/libs/go2idl/go-to-protobuf/protoc-gen-gogo
+
+"${KUBE_ROOT}/hack/after-build/verify-generated-protobuf.sh" "$@"
+
+# ex: ts=2 sw=2 et filetype=sh