Merge pull request #17853 from smarterclayton/non_go_idl

cmd/libs/go2idl/client-gen/generators/generator-for-group.go

@@ -57,12 +57,12 @@ func (g *genGroup) GenerateType(c *generator.Context, t *types.Type, w io.Writer
 		"group":                      g.group,
 		"Group":                      namer.IC(g.group),
 		"types":                      g.types,
-		"Config":                     c.Universe.Get(types.Name{pkgUnversioned, "Config"}),
-		"DefaultKubernetesUserAgent": c.Universe.Get(types.Name{pkgUnversioned, "DefaultKubernetesUserAgent"}),
-		"RESTClient":                 c.Universe.Get(types.Name{pkgUnversioned, "RESTClient"}),
-		"RESTClientFor":              c.Universe.Get(types.Name{pkgUnversioned, "RESTClientFor"}),
-		"latestGroup":                c.Universe.Get(types.Name{pkgLatest, "Group"}),
-		"GroupOrDie":                 c.Universe.Get(types.Name{pkgLatest, "GroupOrDie"}),
+		"Config":                     c.Universe.Get(types.Name{Package: pkgUnversioned, Name: "Config"}),
+		"DefaultKubernetesUserAgent": c.Universe.Get(types.Name{Package: pkgUnversioned, Name: "DefaultKubernetesUserAgent"}),
+		"RESTClient":                 c.Universe.Get(types.Name{Package: pkgUnversioned, Name: "RESTClient"}),
+		"RESTClientFor":              c.Universe.Get(types.Name{Package: pkgUnversioned, Name: "RESTClientFor"}),
+		"latestGroup":                c.Universe.Get(types.Name{Package: pkgLatest, Name: "Group"}),
+		"GroupOrDie":                 c.Universe.Get(types.Name{Package: pkgLatest, Name: "GroupOrDie"}),
 	}
 	sw.Do(groupInterfaceTemplate, m)
 	sw.Do(groupClientTemplate, m)

cmd/libs/go2idl/client-gen/generators/generator-for-type.go

@@ -54,11 +54,11 @@ func (g *genClientForType) GenerateType(c *generator.Context, t *types.Type, w i
 		"type":             t,
 		"package":          pkg,
 		"Package":          namer.IC(pkg),
-		"fieldSelector":    c.Universe.Get(types.Name{"k8s.io/kubernetes/pkg/fields", "Selector"}),
-		"labelSelector":    c.Universe.Get(types.Name{"k8s.io/kubernetes/pkg/labels", "Selector"}),
-		"watchInterface":   c.Universe.Get(types.Name{"k8s.io/kubernetes/pkg/watch", "Interface"}),
-		"apiDeleteOptions": c.Universe.Get(types.Name{"k8s.io/kubernetes/pkg/api", "DeleteOptions"}),
-		"apiListOptions":   c.Universe.Get(types.Name{"k8s.io/kubernetes/pkg/api", "ListOptions"}),
+		"fieldSelector":    c.Universe.Get(types.Name{Package: "k8s.io/kubernetes/pkg/fields", Name: "Selector"}),
+		"labelSelector":    c.Universe.Get(types.Name{Package: "k8s.io/kubernetes/pkg/labels", Name: "Selector"}),
+		"watchInterface":   c.Universe.Get(types.Name{Package: "k8s.io/kubernetes/pkg/watch", Name: "Interface"}),
+		"apiDeleteOptions": c.Universe.Get(types.Name{Package: "k8s.io/kubernetes/pkg/api", Name: "DeleteOptions"}),
+		"apiListOptions":   c.Universe.Get(types.Name{Package: "k8s.io/kubernetes/pkg/api", Name: "ListOptions"}),
 	}
 	sw.Do(namespacerTemplate, m)
 	sw.Do(interfaceTemplate, m)

cmd/libs/go2idl/generator/default_generator.go

@@ -23,6 +23,10 @@ import (
 	"k8s.io/kubernetes/cmd/libs/go2idl/types"
 )
 
+const (
+	GolangFileType = "golang"
+)
+
 // DefaultGen implements a do-nothing Generator.
 //
 // It can be used to implement static content files.
@@ -45,6 +49,7 @@ func (d DefaultGen) PackageVars(*Context) []string                       { retur
 func (d DefaultGen) PackageConsts(*Context) []string                     { return []string{} }
 func (d DefaultGen) GenerateType(*Context, *types.Type, io.Writer) error { return nil }
 func (d DefaultGen) Filename() string                                    { return d.OptionalName + ".go" }
+func (d DefaultGen) FileType() string                                    { return GolangFileType }
 
 func (d DefaultGen) Init(c *Context, w io.Writer) error {
 	_, err := w.Write(d.OptionalBody)

cmd/libs/go2idl/generator/execute.go

@@ -44,17 +44,9 @@ func (c *Context) ExecutePackages(outDir string, packages Packages) error {
 	return nil
 }
 
-type file struct {
-	name        string
-	packageName string
-	header      []byte
-	imports     map[string]struct{}
-	vars        bytes.Buffer
-	consts      bytes.Buffer
-	body        bytes.Buffer
-}
+type golangFileType struct{}
 
-func (f *file) assembleToFile(pathname string) error {
+func (ft golangFileType) AssembleFile(f *File, pathname string) error {
 	log.Printf("Assembling file %q", pathname)
 	destFile, err := os.Create(pathname)
 	if err != nil {
@@ -64,7 +56,7 @@ func (f *file) assembleToFile(pathname string) error {
 
 	b := &bytes.Buffer{}
 	et := NewErrorTracker(b)
-	f.assemble(et)
+	ft.assemble(et, f)
 	if et.Error() != nil {
 		return et.Error()
 	}
@@ -78,14 +70,14 @@ func (f *file) assembleToFile(pathname string) error {
 	}
 }
 
-func (f *file) assemble(w io.Writer) {
-	w.Write(f.header)
-	fmt.Fprintf(w, "package %v\n\n", f.packageName)
+func (ft golangFileType) assemble(w io.Writer, f *File) {
+	w.Write(f.Header)
+	fmt.Fprintf(w, "package %v\n\n", f.PackageName)
 
-	if len(f.imports) > 0 {
+	if len(f.Imports) > 0 {
 		fmt.Fprint(w, "import (\n")
 		// TODO: sort imports like goimports does.
-		for i := range f.imports {
+		for i := range f.Imports {
 			if strings.Contains(i, "\"") {
 				// they included quotes, or are using the
 				// `name "path/to/pkg"` format.
@@ -97,27 +89,27 @@ func (f *file) assemble(w io.Writer) {
 		fmt.Fprint(w, ")\n\n")
 	}
 
-	if f.vars.Len() > 0 {
+	if f.Vars.Len() > 0 {
 		fmt.Fprint(w, "var (\n")
-		w.Write(f.vars.Bytes())
+		w.Write(f.Vars.Bytes())
 		fmt.Fprint(w, ")\n\n")
 	}
 
-	if f.consts.Len() > 0 {
+	if f.Consts.Len() > 0 {
 		fmt.Fprint(w, "const (\n")
-		w.Write(f.consts.Bytes())
+		w.Write(f.Consts.Bytes())
 		fmt.Fprint(w, ")\n\n")
 	}
 
-	w.Write(f.body.Bytes())
+	w.Write(f.Body.Bytes())
 }
 
 // format should be one line only, and not end with \n.
 func addIndentHeaderComment(b *bytes.Buffer, format string, args ...interface{}) {
 	if b.Len() > 0 {
-		fmt.Fprintf(b, "\n\t// "+format+"\n", args...)
+		fmt.Fprintf(b, "\n// "+format+"\n", args...)
 	} else {
-		fmt.Fprintf(b, "\t// "+format+"\n", args...)
+		fmt.Fprintf(b, "// "+format+"\n", args...)
 	}
 }
 
@@ -161,52 +153,66 @@ func (c *Context) ExecutePackage(outDir string, p Package) error {
 	// Filter out any types the *package* doesn't care about.
 	packageContext := c.filteredBy(p.Filter)
 	os.MkdirAll(path, 0755)
-	files := map[string]*file{}
+	files := map[string]*File{}
 	for _, g := range p.Generators(packageContext) {
 		// Filter out types the *generator* doesn't care about.
 		genContext := packageContext.filteredBy(g.Filter)
 		// Now add any extra name systems defined by this generator
 		genContext = genContext.addNameSystems(g.Namers(genContext))
 
+		fileType := g.FileType()
+		if len(fileType) == 0 {
+			return fmt.Errorf("generator %q must specify a file type", g.Name())
+		}
 		f := files[g.Filename()]
 		if f == nil {
 			// This is the first generator to reference this file, so start it.
-			f = &file{
-				name:        g.Filename(),
-				packageName: p.Name(),
-				header:      p.Header(g.Filename()),
-				imports:     map[string]struct{}{},
+			f = &File{
+				Name:        g.Filename(),
+				FileType:    fileType,
+				PackageName: p.Name(),
+				Header:      p.Header(g.Filename()),
+				Imports:     map[string]struct{}{},
+			}
+			files[f.Name] = f
+		} else {
+			if f.FileType != g.FileType() {
+				return fmt.Errorf("file %q already has type %q, but generator %q wants to use type %q", f.Name, f.FileType, g.Name(), g.FileType())
 			}
-			files[f.name] = f
 		}
+
 		if vars := g.PackageVars(genContext); len(vars) > 0 {
-			addIndentHeaderComment(&f.vars, "Package-wide variables from generator %q.", g.Name())
+			addIndentHeaderComment(&f.Vars, "Package-wide variables from generator %q.", g.Name())
 			for _, v := range vars {
-				if _, err := fmt.Fprintf(&f.vars, "\t%s\n", v); err != nil {
+				if _, err := fmt.Fprintf(&f.Vars, "%s\n", v); err != nil {
 					return err
 				}
 			}
 		}
 		if consts := g.PackageVars(genContext); len(consts) > 0 {
-			addIndentHeaderComment(&f.consts, "Package-wide consts from generator %q.", g.Name())
+			addIndentHeaderComment(&f.Consts, "Package-wide consts from generator %q.", g.Name())
 			for _, v := range consts {
-				if _, err := fmt.Fprintf(&f.consts, "\t%s\n", v); err != nil {
+				if _, err := fmt.Fprintf(&f.Consts, "%s\n", v); err != nil {
 					return err
 				}
 			}
 		}
-		if err := genContext.executeBody(&f.body, g); err != nil {
+		if err := genContext.executeBody(&f.Body, g); err != nil {
 			return err
 		}
 		if imports := g.Imports(genContext); len(imports) > 0 {
 			for _, i := range imports {
-				f.imports[i] = struct{}{}
+				f.Imports[i] = struct{}{}
 			}
 		}
 	}
 
 	for _, f := range files {
-		if err := f.assembleToFile(filepath.Join(path, f.name)); err != nil {
+		assembler, ok := c.FileTypes[f.FileType]
+		if !ok {
+			return fmt.Errorf("the file type %q registered for file %q does not exist in the context", f.FileType, f.Name)
+		}
+		if err := assembler.AssembleFile(f, filepath.Join(path, f.Name)); err != nil {
 			return err
 		}
 	}

cmd/libs/go2idl/generator/generator.go

@@ -17,6 +17,7 @@ limitations under the License.
 package generator
 
 import (
+	"bytes"
 	"io"
 
 	"k8s.io/kubernetes/cmd/libs/go2idl/namer"
@@ -48,6 +49,21 @@ type Package interface {
 	Generators(*Context) []Generator
 }
 
+type File struct {
+	Name        string
+	FileType    string
+	PackageName string
+	Header      []byte
+	Imports     map[string]struct{}
+	Vars        bytes.Buffer
+	Consts      bytes.Buffer
+	Body        bytes.Buffer
+}
+
+type FileType interface {
+	AssembleFile(f *File, path string) error
+}
+
 // Packages is a list of packages to generate.
 type Packages []Package
 
@@ -120,6 +136,10 @@ type Generator interface {
 	// TODO: provide per-file import tracking, removing the requirement
 	// that generators coordinate..
 	Filename() string
+
+	// A registered file type in the context to generate this file with. If
+	// the FileType is not found in the context, execution will stop.
+	FileType() string
 }
 
 // Context is global context for individual generators to consume.
@@ -134,6 +154,10 @@ type Context struct {
 	// The canonical ordering of the types (will be filtered by both the
 	// Package's and Generator's Filter methods).
 	Order []*types.Type
+
+	// A set of types this context can process. If this is empty or nil,
+	// the default "golang" filetype will be provided.
+	FileTypes map[string]FileType
 }
 
 // NewContext generates a context from the given builder, naming systems, and
@@ -147,6 +171,9 @@ func NewContext(b *parser.Builder, nameSystems namer.NameSystems, canonicalOrder
 	c := &Context{
 		Namers:   namer.NameSystems{},
 		Universe: u,
+		FileTypes: map[string]FileType{
+			GolangFileType: golangFileType{},
+		},
 	}
 
 	for name, systemNamer := range nameSystems {

cmd/libs/go2idl/generator/snippet_writer.go

@@ -116,6 +116,10 @@ func (s *SnippetWriter) Do(format string, args interface{}) *SnippetWriter {
 	return s
 }
 
+func (s *SnippetWriter) Out() io.Writer {
+	return s.w
+}
+
 // Error returns any encountered error.
 func (s *SnippetWriter) Error() error {
 	return s.err

cmd/libs/go2idl/namer/namer_test.go

@@ -27,26 +27,26 @@ func TestNameStrategy(t *testing.T) {
 	u := types.Universe{}
 
 	// Add some types.
-	base := u.Get(types.Name{"foo/bar", "Baz"})
+	base := u.Get(types.Name{Package: "foo/bar", Name: "Baz"})
 	base.Kind = types.Struct
 
-	tmp := u.Get(types.Name{"", "[]bar.Baz"})
+	tmp := u.Get(types.Name{Package: "", Name: "[]bar.Baz"})
 	tmp.Kind = types.Slice
 	tmp.Elem = base
 
-	tmp = u.Get(types.Name{"", "map[string]bar.Baz"})
+	tmp = u.Get(types.Name{Package: "", Name: "map[string]bar.Baz"})
 	tmp.Kind = types.Map
 	tmp.Key = types.String
 	tmp.Elem = base
 
-	tmp = u.Get(types.Name{"foo/other", "Baz"})
+	tmp = u.Get(types.Name{Package: "foo/other", Name: "Baz"})
 	tmp.Kind = types.Struct
 	tmp.Members = []types.Member{{
 		Embedded: true,
 		Type:     base,
 	}}
 
-	u.Get(types.Name{"", "string"})
+	u.Get(types.Name{Package: "", Name: "string"})
 
 	o := Orderer{NewPublicNamer(0)}
 	order := o.Order(u)

cmd/libs/go2idl/parser/parse.go

@@ -83,6 +83,11 @@ func New() *Builder {
 	}
 }
 
+// AddBuildTags adds the specified build tags to the parse context.
+func (b *Builder) AddBuildTags(tags ...string) {
+	b.context.BuildTags = append(b.context.BuildTags, tags...)
+}
+
 // Get package information from the go/build package. Automatically excludes
 // e.g. test files and files for other platforms-- there is quite a bit of
 // logic of that nature in the build package.

cmd/libs/go2idl/parser/parse_test.go

@@ -175,7 +175,7 @@ type Blah struct {
 
 	_, u, o := construct(t, structTest, namer.NewPublicNamer(0))
 	t.Logf("%#v", o)
-	blahT := u.Get(types.Name{"base/foo/proto", "Blah"})
+	blahT := u.Get(types.Name{Package: "base/foo/proto", Name: "Blah"})
 	if blahT == nil {
 		t.Fatal("type not found")
 	}
@@ -344,11 +344,11 @@ type Interface interface{Method(a, b string) (c, d string)}
 		}
 
 		// Also do some one-off checks
-		gtest := u.Get(types.Name{"g", "Test"})
+		gtest := u.Get(types.Name{Package: "g", Name: "Test"})
 		if e, a := 1, len(gtest.Methods); e != a {
 			t.Errorf("expected %v but found %v methods: %#v", e, a, gtest)
 		}
-		iface := u.Get(types.Name{"g", "Interface"})
+		iface := u.Get(types.Name{Package: "g", Name: "Interface"})
 		if e, a := 1, len(iface.Methods); e != a {
 			t.Errorf("expected %v but found %v methods: %#v", e, a, iface)
 		}

cmd/libs/go2idl/types/flatten_test.go

@@ -23,7 +23,7 @@ import (
 
 func TestFlatten(t *testing.T) {
 	mapType := &Type{
-		Name: Name{"", "map[string]string"},
+		Name: Name{Package: "", Name: "map[string]string"},
 		Kind: Map,
 		Key:  String,
 		Elem: String,
@@ -33,7 +33,7 @@ func TestFlatten(t *testing.T) {
 			Name:     "Baz",
 			Embedded: true,
 			Type: &Type{
-				Name: Name{"pkg", "Baz"},
+				Name: Name{Package: "pkg", Name: "Baz"},
 				Kind: Struct,
 				Members: []Member{
 					{Name: "Foo", Type: String},
@@ -41,7 +41,7 @@ func TestFlatten(t *testing.T) {
 						Name:     "Qux",
 						Embedded: true,
 						Type: &Type{
-							Name:    Name{"pkg", "Qux"},
+							Name:    Name{Package: "pkg", Name: "Qux"},
 							Kind:    Struct,
 							Members: []Member{{Name: "Zot", Type: String}},
 						},

cmd/libs/go2idl/types/types.go

@@ -18,10 +18,13 @@ package types
 
 // A type name may have a package qualifier.
 type Name struct {
-	// Empty if embedded or builtin. This is the package path.
+	// Empty if embedded or builtin. This is the package path unless Path is specified.
 	Package string
 	// The type name.
 	Name string
+	// An optional location of the type definition for languages that can have disjoint
+	// packages and paths.
+	Path string
 }
 
 // String returns the name formatted as a string.
@@ -104,7 +107,7 @@ func (p *Package) Get(typeName string) *Type {
 			return t
 		}
 	}
-	t := &Type{Name: Name{p.Path, typeName}}
+	t := &Type{Name: Name{Package: p.Path, Name: typeName}}
 	p.Types[typeName] = t
 	return t
 }
@@ -280,6 +283,22 @@ var (
 		Name: Name{Name: "uint"},
 		Kind: Builtin,
 	}
+	Uintptr = &Type{
+		Name: Name{Name: "uintptr"},
+		Kind: Builtin,
+	}
+	Float64 = &Type{
+		Name: Name{Name: "float64"},
+		Kind: Builtin,
+	}
+	Float32 = &Type{
+		Name: Name{Name: "float32"},
+		Kind: Builtin,
+	}
+	Float = &Type{
+		Name: Name{Name: "float"},
+		Kind: Builtin,
+	}
 	Bool = &Type{
 		Name: Name{Name: "bool"},
 		Kind: Builtin,
@@ -291,19 +310,23 @@ var (
 
 	builtins = &Package{
 		Types: map[string]*Type{
-			"bool":   Bool,
-			"string": String,
-			"int":    Int,
-			"int64":  Int64,
-			"int32":  Int32,
-			"int16":  Int16,
-			"int8":   Byte,
-			"uint":   Uint,
-			"uint64": Uint64,
-			"uint32": Uint32,
-			"uint16": Uint16,
-			"uint8":  Byte,
-			"byte":   Byte,
+			"bool":    Bool,
+			"string":  String,
+			"int":     Int,
+			"int64":   Int64,
+			"int32":   Int32,
+			"int16":   Int16,
+			"int8":    Byte,
+			"uint":    Uint,
+			"uint64":  Uint64,
+			"uint32":  Uint32,
+			"uint16":  Uint16,
+			"uint8":   Byte,
+			"uintptr": Uintptr,
+			"byte":    Byte,
+			"float":   Float,
+			"float64": Float64,
+			"float32": Float32,
 		},
 		Imports: map[string]*Package{},
 		Path:    "",

cmd/libs/go2idl/types/types_test.go

@@ -25,7 +25,7 @@ func TestGetBuiltin(t *testing.T) {
 	if builtinPkg := u.Package(""); builtinPkg.Has("string") {
 		t.Errorf("Expected builtin package to not have builtins until they're asked for explicitly. %#v", builtinPkg)
 	}
-	s := u.Get(Name{"", "string"})
+	s := u.Get(Name{Package: "", Name: "string"})
 	if s != String {
 		t.Errorf("Expected canonical string type.")
 	}
@@ -39,7 +39,7 @@ func TestGetBuiltin(t *testing.T) {
 
 func TestGetMarker(t *testing.T) {
 	u := Universe{}
-	n := Name{"path/to/package", "Foo"}
+	n := Name{Package: "path/to/package", Name: "Foo"}
 	f := u.Get(n)
 	if f == nil || f.Name != n {
 		t.Errorf("Expected marker type.")