Skip to content

K
k3s
Commit 9c74026f authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #46803 from apelisse/new-download-openapi

Automatic merge from submit-queue (batch tested with PRs 43558, 48261, 42376, 46803, 47058) OpenAPI downloads protobuf rather than Json **What this PR does / why we need it**: The current implementation of the OpenAPI getter fetches the swagger in a Json format from the apiserver. The Json file is big (~1.7mb), which means that it takes a long time to download, and then a long time to parse. Because that is going to be needed on each `kubectl` run later, we want this to be as fast as possible. The apiserver has been modified to be able to return a protobuf version of the swagger, which this patch intends to use. Note that there is currently no piece of code that exists that allows us to go from the protobuf version of the file, back into Json and/or `spec.Swagger`. Because the protobuf is not very different (but significantly different enough that it can't be translated), I've updated the code to use `openapi_v2.Document` (the protobuf type) everywhere rather than `spec.Swagger`. The behavior should be identical though. There are more changes that are coming in follow-up pull-requests: using the gzip version (also provided by the new apiserver) to even further reduce the size of the downloaded content, and use the HTTP Etag cache mechanism to completely get rid of recurrent fetch requests. I'm currently working on these two features. **Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: partly #38637 **Special notes for your reviewer**: **Release note**: ```release-note NONE ```
parents 87c6fb5d 224dba9a
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 192 additions and 150 deletions
pkg/kubectl/cmd/util/BUILD
View file @ 9c74026f
......@@ -46,8 +46,8 @@ go_library(
"//pkg/version:go_default_library",
"//vendor/github.com/emicklei/go-restful-swagger12:go_default_library",
"//vendor/github.com/evanphx/json-patch:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library",
"//vendor/github.com/spf13/cobra:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
......@@ -108,7 +108,7 @@ go_test(
"//pkg/kubectl/resource:go_default_library",
"//pkg/util/exec:go_default_library",
"//vendor/github.com/emicklei/go-restful-swagger12:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
......
pkg/kubectl/cmd/util/cached_discovery.go
View file @ 9c74026f
......@@ -25,8 +25,8 @@ import (
"time"
"github.com/emicklei/go-restful-swagger12"
"github.com/go-openapi/spec"
"github.com/golang/glog"
"github.com/googleapis/gnostic/OpenAPIv2"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
......@@ -237,7 +237,7 @@ func (d *CachedDiscoveryClient) SwaggerSchema(version schema.GroupVersion) (*swa
return d.delegate.SwaggerSchema(version)
}
func (d *CachedDiscoveryClient) OpenAPISchema() (*spec.Swagger, error) {
func (d *CachedDiscoveryClient) OpenAPISchema() (*openapi_v2.Document, error) {
return d.delegate.OpenAPISchema()
}
......
pkg/kubectl/cmd/util/cached_discovery_test.go
View file @ 9c74026f
......@@ -23,7 +23,7 @@ import (
"time"
"github.com/emicklei/go-restful-swagger12"
"github.com/go-openapi/spec"
"github.com/googleapis/gnostic/OpenAPIv2"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/api/errors"
......@@ -171,7 +171,7 @@ func (c *fakeDiscoveryClient) SwaggerSchema(version schema.GroupVersion) (*swagg
return &swagger.ApiDeclaration{}, nil
}
func (c *fakeDiscoveryClient) OpenAPISchema() (*spec.Swagger, error) {
func (c *fakeDiscoveryClient) OpenAPISchema() (*openapi_v2.Document, error) {
c.openAPICalls = c.openAPICalls + 1
return &spec.Swagger{}, nil
return &openapi_v2.Document{}, nil
}
pkg/kubectl/cmd/util/openapi/BUILD
View file @ 9c74026f
......@@ -22,6 +22,8 @@ go_library(
"//pkg/version:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library",
"//vendor/gopkg.in/yaml.v2:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/client-go/discovery:go_default_library",
......@@ -41,12 +43,14 @@ go_test(
tags = ["automanaged"],
deps = [
"//pkg/kubectl/cmd/util/openapi:go_default_library",
"//vendor/github.com/go-openapi/loads:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library",
"//vendor/github.com/googleapis/gnostic/compiler:go_default_library",
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/github.com/onsi/ginkgo/config:go_default_library",
"//vendor/github.com/onsi/ginkgo/types:go_default_library",
"//vendor/github.com/onsi/gomega:go_default_library",
"//vendor/gopkg.in/yaml.v2:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
],
)
......
pkg/kubectl/cmd/util/openapi/openapi_cache.go
View file @ 9c74026f
......@@ -187,7 +187,6 @@ func linkFiles(old, new string) error {
// registerBinaryEncodingTypes registers the types so they can be binary encoded by gob
func registerBinaryEncodingTypes() {
gob.Register(map[string]interface{}{})
gob.Register(map[interface{}]interface{}{})
gob.Register([]interface{}{})
gob.Register(Resources{})
}
pkg/kubectl/cmd/util/openapi/openapi_cache_test.go
View file @ 9c74026f
......@@ -23,8 +23,10 @@ import (
"path/filepath"
"sync"
"github.com/go-openapi/loads"
"github.com/go-openapi/spec"
"gopkg.in/yaml.v2"
"github.com/googleapis/gnostic/OpenAPIv2"
"github.com/googleapis/gnostic/compiler"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
......@@ -220,7 +222,7 @@ type fakeOpenAPIClient struct {
err error
}
func (f *fakeOpenAPIClient) OpenAPISchema() (*spec.Swagger, error) {
func (f *fakeOpenAPIClient) OpenAPISchema() (*openapi_v2.Document, error) {
f.calls = f.calls + 1
if f.err != nil {
......@@ -235,11 +237,11 @@ var data apiData
type apiData struct {
sync.Once
data *spec.Swagger
data *openapi_v2.Document
err error
}
func (d *apiData) OpenAPISchema() (*spec.Swagger, error) {
func (d *apiData) OpenAPISchema() (*openapi_v2.Document, error) {
d.Do(func() {
// Get the path to the swagger.json file
wd, err := os.Getwd()
......@@ -261,14 +263,18 @@ func (d *apiData) OpenAPISchema() (*spec.Swagger, error) {
d.err = err
return
}
// Load the openapi document
doc, err := loads.Spec(specpath)
spec, err := ioutil.ReadFile(specpath)
if err != nil {
d.err = err
return
}
d.data = doc.Spec()
var info yaml.MapSlice
err = yaml.Unmarshal(spec, &info)
if err != nil {
d.err = err
return
}
d.data, d.err = openapi_v2.NewDocument(info, compiler.NewContext("$root", nil))
})
return d.data, d.err
}
staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json
View file @ 9c74026f
......@@ -107,10 +107,6 @@
"Rev": "73d445a93680fa1a78ae23a5839bad48f32ba1ee"
},
{
"ImportPath": "github.com/go-openapi/analysis",
"Rev": "b44dc874b601d9e4e2f6e19140e794ba24bead3b"
},
{
"ImportPath": "github.com/go-openapi/jsonpointer",
"Rev": "46af16f9f7b149af66e5d1bd010e3574dc06de98"
},
......@@ -119,10 +115,6 @@
"Rev": "13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272"
},
{
"ImportPath": "github.com/go-openapi/loads",
"Rev": "18441dfa706d924a39a030ee2c3b1d8d81917b38"
},
{
"ImportPath": "github.com/go-openapi/spec",
"Rev": "6aced65f8501fe1217321abf0749d354824ba2ff"
},
......
staging/src/k8s.io/apiserver/Godeps/Godeps.json
View file @ 9c74026f
......@@ -327,10 +327,6 @@
"Rev": "73d445a93680fa1a78ae23a5839bad48f32ba1ee"
},
{
"ImportPath": "github.com/go-openapi/analysis",
"Rev": "b44dc874b601d9e4e2f6e19140e794ba24bead3b"
},
{
"ImportPath": "github.com/go-openapi/jsonpointer",
"Rev": "46af16f9f7b149af66e5d1bd010e3574dc06de98"
},
......@@ -339,10 +335,6 @@
"Rev": "13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272"
},
{
"ImportPath": "github.com/go-openapi/loads",
"Rev": "18441dfa706d924a39a030ee2c3b1d8d81917b38"
},
{
"ImportPath": "github.com/go-openapi/spec",
"Rev": "6aced65f8501fe1217321abf0749d354824ba2ff"
},
......
staging/src/k8s.io/client-go/Godeps/Godeps.json
View file @ 9c74026f
......@@ -99,10 +99,6 @@
"Rev": "73d445a93680fa1a78ae23a5839bad48f32ba1ee"
},
{
"ImportPath": "github.com/go-openapi/analysis",
"Rev": "b44dc874b601d9e4e2f6e19140e794ba24bead3b"
},
{
"ImportPath": "github.com/go-openapi/jsonpointer",
"Rev": "46af16f9f7b149af66e5d1bd010e3574dc06de98"
},
......@@ -111,10 +107,6 @@
"Rev": "13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272"
},
{
"ImportPath": "github.com/go-openapi/loads",
"Rev": "18441dfa706d924a39a030ee2c3b1d8d81917b38"
},
{
"ImportPath": "github.com/go-openapi/spec",
"Rev": "6aced65f8501fe1217321abf0749d354824ba2ff"
},
......@@ -143,10 +135,38 @@
"Rev": "4bd1920723d7b7c925de087aa32e2187708897f7"
},
{
"ImportPath": "github.com/golang/protobuf/ptypes",
"Rev": "4bd1920723d7b7c925de087aa32e2187708897f7"
},
{
"ImportPath": "github.com/golang/protobuf/ptypes/any",
"Rev": "4bd1920723d7b7c925de087aa32e2187708897f7"
},
{
"ImportPath": "github.com/golang/protobuf/ptypes/duration",
"Rev": "4bd1920723d7b7c925de087aa32e2187708897f7"
},
{
"ImportPath": "github.com/golang/protobuf/ptypes/timestamp",
"Rev": "4bd1920723d7b7c925de087aa32e2187708897f7"
},
{
"ImportPath": "github.com/google/gofuzz",
"Rev": "44d81051d367757e1c7c6a5a86423ece9afcf63c"
},
{
"ImportPath": "github.com/googleapis/gnostic/OpenAPIv2",
"Rev": "68f4ded48ba9414dab2ae69b3f0d69971da73aa5"
},
{
"ImportPath": "github.com/googleapis/gnostic/compiler",
"Rev": "68f4ded48ba9414dab2ae69b3f0d69971da73aa5"
},
{
"ImportPath": "github.com/googleapis/gnostic/extensions",
"Rev": "68f4ded48ba9414dab2ae69b3f0d69971da73aa5"
},
{
"ImportPath": "github.com/hashicorp/golang-lru",
"Rev": "a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4"
},
......
staging/src/k8s.io/client-go/discovery/BUILD
View file @ 9c74026f
......@@ -19,9 +19,9 @@ go_library(
tags = ["automanaged"],
deps = [
"//vendor/github.com/emicklei/go-restful-swagger12:go_default_library",
"//vendor/github.com/go-openapi/loads:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/golang/protobuf/proto:go_default_library",
"//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
......@@ -46,7 +46,8 @@ go_test(
tags = ["automanaged"],
deps = [
"//vendor/github.com/emicklei/go-restful-swagger12:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/gogo/protobuf/proto:go_default_library",
"//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
......
staging/src/k8s.io/client-go/discovery/discovery_client.go
View file @ 9c74026f
......@@ -24,9 +24,9 @@ import (
"strings"
"github.com/emicklei/go-restful-swagger12"
"github.com/golang/protobuf/proto"
"github.com/googleapis/gnostic/OpenAPIv2"
"github.com/go-openapi/loads"
"github.com/go-openapi/spec"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
......@@ -97,7 +97,7 @@ type SwaggerSchemaInterface interface {
// OpenAPISchemaInterface has a method to retrieve the open API schema.
type OpenAPISchemaInterface interface {
// OpenAPISchema retrieves and parses the swagger API schema the server supports.
OpenAPISchema() (*spec.Swagger, error)
OpenAPISchema() (*openapi_v2.Document, error)
}
// DiscoveryClient implements the functions that discover server-supported API groups,
......@@ -375,19 +375,18 @@ func (d *DiscoveryClient) SwaggerSchema(version schema.GroupVersion) (*swagger.A
return &schema, nil
}
// OpenAPISchema fetches the open api schema using a rest client and parses the json.
// Warning: this is very expensive (~1.2s)
func (d *DiscoveryClient) OpenAPISchema() (*spec.Swagger, error) {
data, err := d.restClient.Get().AbsPath("/swagger.json").Do().Raw()
// OpenAPISchema fetches the open api schema using a rest client and parses the proto.
func (d *DiscoveryClient) OpenAPISchema() (*openapi_v2.Document, error) {
data, err := d.restClient.Get().AbsPath("/swagger-2.0.0.pb-v1").Do().Raw()
if err != nil {
return nil, err
}
msg := json.RawMessage(data)
doc, err := loads.Analyzed(msg, "")
document := &openapi_v2.Document{}
err = proto.Unmarshal(data, document)
if err != nil {
return nil, err
}
return doc.Spec(), err
return document, nil
}
// withRetries retries the given recovery function in case the groups supported by the server change after ServerGroup() returns.
......
staging/src/k8s.io/client-go/discovery/discovery_client_test.go
View file @ 9c74026f
......@@ -19,14 +19,16 @@ package discovery_test
import (
"encoding/json"
"fmt"
"mime"
"net/http"
"net/http/httptest"
"reflect"
"testing"
"github.com/emicklei/go-restful-swagger12"
"github.com/gogo/protobuf/proto"
"github.com/googleapis/gnostic/OpenAPIv2"
"github.com/go-openapi/spec"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
......@@ -327,30 +329,44 @@ func TestGetSwaggerSchemaFail(t *testing.T) {
}
}
var returnedOpenAPI = spec.Swagger{
SwaggerProps: spec.SwaggerProps{
Definitions: spec.Definitions{
"fake.type.1": spec.Schema{
SchemaProps: spec.SchemaProps{
Properties: map[string]spec.Schema{
"count": {
SchemaProps: spec.SchemaProps{
Type: []string{"integer"},
var returnedOpenAPI = openapi_v2.Document{
Definitions: &openapi_v2.Definitions{
AdditionalProperties: []*openapi_v2.NamedSchema{
{
Name: "fake.type.1",
Value: &openapi_v2.Schema{
Properties: &openapi_v2.Properties{
AdditionalProperties: []*openapi_v2.NamedSchema{
{
Name: "count",
Value: &openapi_v2.Schema{
Type: &openapi_v2.TypeItem{
Value: []string{"integer"},
},
},
},
},
},
},
},
"fake.type.2": spec.Schema{
SchemaProps: spec.SchemaProps{
Properties: map[string]spec.Schema{
"count": {
SchemaProps: spec.SchemaProps{
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
{
Name: "fake.type.2",
Value: &openapi_v2.Schema{
Properties: &openapi_v2.Properties{
AdditionalProperties: []*openapi_v2.NamedSchema{
{
Name: "count",
Value: &openapi_v2.Schema{
Type: &openapi_v2.TypeItem{
Value: []string{"array"},
},
Items: &openapi_v2.ItemsItem{
Schema: []*openapi_v2.Schema{
{
Type: &openapi_v2.TypeItem{
Value: []string{"string"},
},
},
},
},
},
......@@ -366,19 +382,20 @@ var returnedOpenAPI = spec.Swagger{
func openapiSchemaFakeServer() (*httptest.Server, error) {
var sErr error
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.URL.Path != "/swagger.json" {
if req.URL.Path != "/swagger-2.0.0.pb-v1" {
sErr = fmt.Errorf("Unexpected url %v", req.URL)
}
if req.Method != "GET" {
sErr = fmt.Errorf("Unexpected method %v", req.Method)
}
output, err := json.Marshal(returnedOpenAPI)
mime.AddExtensionType(".pb-v1", "application/com.github.googleapis.gnostic.OpenAPIv2@68f4ded+protobuf")
output, err := proto.Marshal(&returnedOpenAPI)
if err != nil {
sErr = err
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(output)
}))
......
staging/src/k8s.io/client-go/discovery/fake/BUILD
View file @ 9c74026f
......@@ -13,7 +13,7 @@ go_library(
tags = ["automanaged"],
deps = [
"//vendor/github.com/emicklei/go-restful-swagger12:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
......
staging/src/k8s.io/client-go/discovery/fake/discovery.go
View file @ 9c74026f
......@@ -20,8 +20,8 @@ import (
"fmt"
"github.com/emicklei/go-restful-swagger12"
"github.com/googleapis/gnostic/OpenAPIv2"
"github.com/go-openapi/spec"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
......@@ -93,7 +93,9 @@ func (c *FakeDiscovery) SwaggerSchema(version schema.GroupVersion) (*swagger.Api
return &swagger.ApiDeclaration{}, nil
}
func (c *FakeDiscovery) OpenAPISchema() (*spec.Swagger, error) { return &spec.Swagger{}, nil }
func (c *FakeDiscovery) OpenAPISchema() (*openapi_v2.Document, error) {
return &openapi_v2.Document{}, nil
}
func (c *FakeDiscovery) RESTClient() restclient.Interface {
return nil
......
staging/src/k8s.io/client-go/discovery/restmapper_test.go
View file @ 9c74026f
......@@ -29,7 +29,7 @@ import (
"k8s.io/client-go/rest/fake"
"github.com/emicklei/go-restful-swagger12"
"github.com/go-openapi/spec"
"github.com/googleapis/gnostic/OpenAPIv2"
"github.com/stretchr/testify/assert"
)
......@@ -348,6 +348,6 @@ func (c *fakeCachedDiscoveryInterface) SwaggerSchema(version schema.GroupVersion
return &swagger.ApiDeclaration{}, nil
}
func (c *fakeCachedDiscoveryInterface) OpenAPISchema() (*spec.Swagger, error) {
return &spec.Swagger{}, nil
func (c *fakeCachedDiscoveryInterface) OpenAPISchema() (*openapi_v2.Document, error) {
return &openapi_v2.Document{}, nil
}
staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json
View file @ 9c74026f
......@@ -115,10 +115,6 @@
"Rev": "73d445a93680fa1a78ae23a5839bad48f32ba1ee"
},
{
"ImportPath": "github.com/go-openapi/analysis",
"Rev": "b44dc874b601d9e4e2f6e19140e794ba24bead3b"
},
{
"ImportPath": "github.com/go-openapi/jsonpointer",
"Rev": "46af16f9f7b149af66e5d1bd010e3574dc06de98"
},
......@@ -127,10 +123,6 @@
"Rev": "13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272"
},
{
"ImportPath": "github.com/go-openapi/loads",
"Rev": "18441dfa706d924a39a030ee2c3b1d8d81917b38"
},
{
"ImportPath": "github.com/go-openapi/spec",
"Rev": "6aced65f8501fe1217321abf0749d354824ba2ff"
},
......
staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json
View file @ 9c74026f
......@@ -107,10 +107,6 @@
"Rev": "73d445a93680fa1a78ae23a5839bad48f32ba1ee"
},
{
"ImportPath": "github.com/go-openapi/analysis",
"Rev": "b44dc874b601d9e4e2f6e19140e794ba24bead3b"
},
{
"ImportPath": "github.com/go-openapi/jsonpointer",
"Rev": "46af16f9f7b149af66e5d1bd010e3574dc06de98"
},
......@@ -119,10 +115,6 @@
"Rev": "13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272"
},
{
"ImportPath": "github.com/go-openapi/loads",
"Rev": "18441dfa706d924a39a030ee2c3b1d8d81917b38"
},
{
"ImportPath": "github.com/go-openapi/spec",
"Rev": "6aced65f8501fe1217321abf0749d354824ba2ff"
},
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment