Commit 6770fa7b authored by Thomas Runyon's avatar Thomas Runyon

Merge remote-tracking branch 'upstream/master' into test-cmd-what

parents 4fc5be8d 939f8ffa
...@@ -3,15 +3,17 @@ ...@@ -3,15 +3,17 @@
"GoVersion": "go1.11", "GoVersion": "go1.11",
"GodepVersion": "v80-k8s-r1", "GodepVersion": "v80-k8s-r1",
"Packages": [ "Packages": [
"github.com/onsi/ginkgo/ginkgo", "github.com/bazelbuild/bazel-gazelle/cmd/gazelle",
"github.com/jteeuwen/go-bindata/go-bindata", "github.com/cespare/prettybench",
"github.com/client9/misspell/cmd/misspell", "github.com/client9/misspell/cmd/misspell",
"github.com/cloudflare/cfssl/cmd/cfssl", "github.com/cloudflare/cfssl/cmd/cfssl",
"github.com/cloudflare/cfssl/cmd/cfssljson", "github.com/cloudflare/cfssl/cmd/cfssljson",
"github.com/bazelbuild/bazel-gazelle/cmd/gazelle", "github.com/jstemmer/go-junit-report",
"github.com/jteeuwen/go-bindata/go-bindata",
"github.com/onsi/ginkgo/ginkgo",
"golang.org/x/lint/golint",
"k8s.io/kube-openapi/cmd/openapi-gen", "k8s.io/kube-openapi/cmd/openapi-gen",
"k8s.io/repo-infra/kazel", "k8s.io/repo-infra/kazel",
"golang.org/x/lint/golint",
"./..." "./..."
], ],
"Deps": [ "Deps": [
...@@ -489,6 +491,10 @@ ...@@ -489,6 +491,10 @@
"Rev": "b38d23b8782a487059e8fc8773e9a5b228a77cb6" "Rev": "b38d23b8782a487059e8fc8773e9a5b228a77cb6"
}, },
{ {
"ImportPath": "github.com/cespare/prettybench",
"Rev": "03b8cfe5406ce67a0b0da46f0c9e78b3d915a2c1"
},
{
"ImportPath": "github.com/chai2010/gettext-go/gettext", "ImportPath": "github.com/chai2010/gettext-go/gettext",
"Rev": "c6fed771bfd517099caf0f7a961671fa8ed08723" "Rev": "c6fed771bfd517099caf0f7a961671fa8ed08723"
}, },
...@@ -2375,6 +2381,18 @@ ...@@ -2375,6 +2381,18 @@
"Rev": "ab8a2e0c74be9d3be70b3184d9acc634935ded82" "Rev": "ab8a2e0c74be9d3be70b3184d9acc634935ded82"
}, },
{ {
"ImportPath": "github.com/jstemmer/go-junit-report",
"Rev": "af01ea7f8024089b458d804d5cdf190f962a9a0c"
},
{
"ImportPath": "github.com/jstemmer/go-junit-report/formatter",
"Rev": "af01ea7f8024089b458d804d5cdf190f962a9a0c"
},
{
"ImportPath": "github.com/jstemmer/go-junit-report/parser",
"Rev": "af01ea7f8024089b458d804d5cdf190f962a9a0c"
},
{
"ImportPath": "github.com/jteeuwen/go-bindata", "ImportPath": "github.com/jteeuwen/go-bindata",
"Comment": "v3.0.7-72-ga0ff2567cfb709", "Comment": "v3.0.7-72-ga0ff2567cfb709",
"Rev": "a0ff2567cfb70903282db057e799fd826784d41d" "Rev": "a0ff2567cfb70903282db057e799fd826784d41d"
...@@ -3624,31 +3642,59 @@ ...@@ -3624,31 +3642,59 @@
}, },
{ {
"ImportPath": "golang.org/x/tools/benchmark/parse", "ImportPath": "golang.org/x/tools/benchmark/parse",
"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32" "Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
}, },
{ {
"ImportPath": "golang.org/x/tools/container/intsets", "ImportPath": "golang.org/x/tools/container/intsets",
"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32" "Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
}, },
{ {
"ImportPath": "golang.org/x/tools/go/ast/astutil", "ImportPath": "golang.org/x/tools/go/ast/astutil",
"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32" "Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
}, },
{ {
"ImportPath": "golang.org/x/tools/go/gcexportdata", "ImportPath": "golang.org/x/tools/go/gcexportdata",
"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32" "Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/go/internal/cgo",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/go/internal/gcimporter",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/go/internal/packagesdriver",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
}, },
{ {
"ImportPath": "golang.org/x/tools/go/gcimporter15", "ImportPath": "golang.org/x/tools/go/packages",
"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32" "Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
}, },
{ {
"ImportPath": "golang.org/x/tools/go/vcs", "ImportPath": "golang.org/x/tools/go/vcs",
"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32" "Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
}, },
{ {
"ImportPath": "golang.org/x/tools/imports", "ImportPath": "golang.org/x/tools/imports",
"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32" "Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/internal/fastwalk",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/internal/gopathwalk",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/internal/module",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/internal/semver",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
}, },
{ {
"ImportPath": "gonum.org/v1/gonum/blas", "ImportPath": "gonum.org/v1/gonum/blas",
......
approvers: approvers:
- castrojo
- chuckbutler
- marcoceppi
- mbruzek
- Cynerva - Cynerva
- ktsakalozos - ktsakalozos
reviewers: reviewers:
- chuckbutler
- marcoceppi
- mbruzek
- thockin - thockin
- mikedanese - mikedanese
- eparis - eparis
......
...@@ -176,6 +176,7 @@ ...@@ -176,6 +176,7 @@
./test/cmd/core.sh ./test/cmd/core.sh
./test/cmd/crd.sh ./test/cmd/crd.sh
./test/cmd/create.sh ./test/cmd/create.sh
./test/cmd/delete.sh
./test/cmd/diff.sh ./test/cmd/diff.sh
./test/cmd/discovery.sh ./test/cmd/discovery.sh
./test/cmd/generic-resources.sh ./test/cmd/generic-resources.sh
......
...@@ -55,15 +55,17 @@ fi ...@@ -55,15 +55,17 @@ fi
# Some things we want in godeps aren't code dependencies, so ./... # Some things we want in godeps aren't code dependencies, so ./...
# won't pick them up. # won't pick them up.
REQUIRED_BINS=( REQUIRED_BINS=(
"github.com/onsi/ginkgo/ginkgo" "github.com/bazelbuild/bazel-gazelle/cmd/gazelle"
"github.com/jteeuwen/go-bindata/go-bindata" "github.com/cespare/prettybench"
"github.com/client9/misspell/cmd/misspell" "github.com/client9/misspell/cmd/misspell"
"github.com/cloudflare/cfssl/cmd/cfssl" "github.com/cloudflare/cfssl/cmd/cfssl"
"github.com/cloudflare/cfssl/cmd/cfssljson" "github.com/cloudflare/cfssl/cmd/cfssljson"
"github.com/bazelbuild/bazel-gazelle/cmd/gazelle" "github.com/jstemmer/go-junit-report"
"github.com/jteeuwen/go-bindata/go-bindata"
"github.com/onsi/ginkgo/ginkgo"
"golang.org/x/lint/golint"
"k8s.io/kube-openapi/cmd/openapi-gen" "k8s.io/kube-openapi/cmd/openapi-gen"
"k8s.io/repo-infra/kazel" "k8s.io/repo-infra/kazel"
"golang.org/x/lint/golint"
"./..." "./..."
) )
......
...@@ -33,7 +33,7 @@ retry() { ...@@ -33,7 +33,7 @@ retry() {
export PATH=${GOPATH}/bin:${PWD}/third_party/etcd:/usr/local/go/bin:${PATH} export PATH=${GOPATH}/bin:${PWD}/third_party/etcd:/usr/local/go/bin:${PATH}
retry go get github.com/cespare/prettybench go install k8s.io/kubernetes/vendor/github.com/cespare/prettybench
# Disable the Go race detector. # Disable the Go race detector.
export KUBE_RACE=" " export KUBE_RACE=" "
......
...@@ -32,7 +32,7 @@ export PATH=${GOPATH}/bin:${HOME}/third_party/etcd:/usr/local/go/bin:${PATH} ...@@ -32,7 +32,7 @@ export PATH=${GOPATH}/bin:${HOME}/third_party/etcd:/usr/local/go/bin:${PATH}
# Install a few things needed by unit and /integration tests. # Install a few things needed by unit and /integration tests.
command -v etcd &>/dev/null || ./hack/install-etcd.sh command -v etcd &>/dev/null || ./hack/install-etcd.sh
go get -u github.com/jstemmer/go-junit-report go install k8s.io/kubernetes/vendor/github.com/jstemmer/go-junit-report
# Enable the Go race detector. # Enable the Go race detector.
export KUBE_RACE=-race export KUBE_RACE=-race
......
...@@ -33,7 +33,7 @@ retry() { ...@@ -33,7 +33,7 @@ retry() {
export PATH=${GOPATH}/bin:${PWD}/third_party/etcd:/usr/local/go/bin:${PATH} export PATH=${GOPATH}/bin:${PWD}/third_party/etcd:/usr/local/go/bin:${PATH}
retry go get github.com/jstemmer/go-junit-report go install k8s.io/kubernetes/vendor/github.com/jstemmer/go-junit-report
# Enable the Go race detector. # Enable the Go race detector.
export KUBE_RACE=-race export KUBE_RACE=-race
......
...@@ -4725,11 +4725,7 @@ func ValidateResourceRequirements(requirements *core.ResourceRequirements, fldPa ...@@ -4725,11 +4725,7 @@ func ValidateResourceRequirements(requirements *core.ResourceRequirements, fldPa
allErrs = append(allErrs, ValidateResourceQuantityValue(string(resourceName), quantity, fldPath)...) allErrs = append(allErrs, ValidateResourceQuantityValue(string(resourceName), quantity, fldPath)...)
if helper.IsHugePageResourceName(resourceName) { if helper.IsHugePageResourceName(resourceName) {
if !utilfeature.DefaultFeatureGate.Enabled(features.HugePages) { limContainsHugePages = true
allErrs = append(allErrs, field.Forbidden(limPath, fmt.Sprintf("%s field disabled by feature-gate for ResourceRequirements", resourceName)))
} else {
limContainsHugePages = true
}
} }
if supportedQoSComputeResources.Has(string(resourceName)) { if supportedQoSComputeResources.Has(string(resourceName)) {
......
...@@ -53,8 +53,6 @@ const ( ...@@ -53,8 +53,6 @@ const (
VolDir = "kubevols" VolDir = "kubevols"
RoundTripperDefaultCount = 3 RoundTripperDefaultCount = 3
DummyVMPrefixName = "vsphere-k8s" DummyVMPrefixName = "vsphere-k8s"
MacOuiVC = "00:50:56"
MacOuiEsx = "00:0c:29"
CleanUpDummyVMRoutineInterval = 5 CleanUpDummyVMRoutineInterval = 5
) )
...@@ -533,6 +531,15 @@ func (vs *VSphere) Instances() (cloudprovider.Instances, bool) { ...@@ -533,6 +531,15 @@ func (vs *VSphere) Instances() (cloudprovider.Instances, bool) {
} }
func getLocalIP() ([]v1.NodeAddress, error) { func getLocalIP() ([]v1.NodeAddress, error) {
// hashtable with VMware-allocated OUIs for MAC filtering
// List of official OUIs: http://standards-oui.ieee.org/oui.txt
vmwareOUI := map[string]bool{
"00:05:69": true,
"00:0c:29": true,
"00:1c:14": true,
"00:50:56": true,
}
addrs := []v1.NodeAddress{} addrs := []v1.NodeAddress{}
ifaces, err := net.Interfaces() ifaces, err := net.Interfaces()
if err != nil { if err != nil {
...@@ -548,9 +555,12 @@ func getLocalIP() ([]v1.NodeAddress, error) { ...@@ -548,9 +555,12 @@ func getLocalIP() ([]v1.NodeAddress, error) {
if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To4() != nil { if ipnet.IP.To4() != nil {
// Filter external IP by MAC address OUIs from vCenter and from ESX // Filter external IP by MAC address OUIs from vCenter and from ESX
var addressType v1.NodeAddressType vmMACAddr := strings.ToLower(i.HardwareAddr.String())
if strings.HasPrefix(i.HardwareAddr.String(), MacOuiVC) || // Making sure that the MAC address is long enough
strings.HasPrefix(i.HardwareAddr.String(), MacOuiEsx) { if len(vmMACAddr) < 17 {
return addrs, fmt.Errorf("MAC address %q is invalid", vmMACAddr)
}
if vmwareOUI[vmMACAddr[:8]] {
nodehelpers.AddToNodeAddresses(&addrs, nodehelpers.AddToNodeAddresses(&addrs,
v1.NodeAddress{ v1.NodeAddress{
Type: v1.NodeExternalIP, Type: v1.NodeExternalIP,
...@@ -561,8 +571,10 @@ func getLocalIP() ([]v1.NodeAddress, error) { ...@@ -561,8 +571,10 @@ func getLocalIP() ([]v1.NodeAddress, error) {
Address: ipnet.IP.String(), Address: ipnet.IP.String(),
}, },
) )
klog.V(4).Infof("Detected local IP address as %q", ipnet.IP.String())
} else {
klog.Warningf("Failed to patch IP as MAC address %q does not belong to a VMware platform", vmMACAddr)
} }
klog.V(4).Infof("Find local IP address %v and set type to %v", ipnet.IP.String(), addressType)
} }
} }
} }
......
...@@ -11,6 +11,7 @@ go_library( ...@@ -11,6 +11,7 @@ go_library(
srcs = [ srcs = [
"doc.go", "doc.go",
"endpoints_controller.go", "endpoints_controller.go",
"trigger_time_tracker.go",
], ],
importpath = "k8s.io/kubernetes/pkg/controller/endpoint", importpath = "k8s.io/kubernetes/pkg/controller/endpoint",
deps = [ deps = [
...@@ -39,7 +40,10 @@ go_library( ...@@ -39,7 +40,10 @@ go_library(
go_test( go_test(
name = "go_default_test", name = "go_default_test",
srcs = ["endpoints_controller_test.go"], srcs = [
"endpoints_controller_test.go",
"trigger_time_tracker_test.go",
],
embed = [":go_default_library"], embed = [":go_default_library"],
deps = [ deps = [
"//pkg/api/testapi:go_default_library", "//pkg/api/testapi:go_default_library",
......
...@@ -101,6 +101,8 @@ func NewEndpointController(podInformer coreinformers.PodInformer, serviceInforme ...@@ -101,6 +101,8 @@ func NewEndpointController(podInformer coreinformers.PodInformer, serviceInforme
e.endpointsLister = endpointsInformer.Lister() e.endpointsLister = endpointsInformer.Lister()
e.endpointsSynced = endpointsInformer.Informer().HasSynced e.endpointsSynced = endpointsInformer.Informer().HasSynced
e.triggerTimeTracker = NewTriggerTimeTracker()
return e return e
} }
...@@ -138,6 +140,10 @@ type EndpointController struct { ...@@ -138,6 +140,10 @@ type EndpointController struct {
// workerLoopPeriod is the time between worker runs. The workers process the queue of service and pod changes. // workerLoopPeriod is the time between worker runs. The workers process the queue of service and pod changes.
workerLoopPeriod time.Duration workerLoopPeriod time.Duration
// triggerTimeTracker is an util used to compute and export the EndpointsLastChangeTriggerTime
// annotation.
triggerTimeTracker *TriggerTimeTracker
} }
// Run will not return until stopCh is closed. workers determines how many // Run will not return until stopCh is closed. workers determines how many
...@@ -399,6 +405,7 @@ func (e *EndpointController) syncService(key string) error { ...@@ -399,6 +405,7 @@ func (e *EndpointController) syncService(key string) error {
if err != nil && !errors.IsNotFound(err) { if err != nil && !errors.IsNotFound(err) {
return err return err
} }
e.triggerTimeTracker.DeleteEndpoints(namespace, name)
return nil return nil
} }
...@@ -427,6 +434,12 @@ func (e *EndpointController) syncService(key string) error { ...@@ -427,6 +434,12 @@ func (e *EndpointController) syncService(key string) error {
} }
} }
// We call ComputeEndpointsLastChangeTriggerTime here to make sure that the state of the trigger
// time tracker gets updated even if the sync turns out to be no-op and we don't update the
// endpoints object.
endpointsLastChangeTriggerTime := e.triggerTimeTracker.
ComputeEndpointsLastChangeTriggerTime(namespace, name, service, pods)
subsets := []v1.EndpointSubset{} subsets := []v1.EndpointSubset{}
var totalReadyEps int var totalReadyEps int
var totalNotReadyEps int var totalNotReadyEps int
...@@ -506,6 +519,11 @@ func (e *EndpointController) syncService(key string) error { ...@@ -506,6 +519,11 @@ func (e *EndpointController) syncService(key string) error {
newEndpoints.Annotations = make(map[string]string) newEndpoints.Annotations = make(map[string]string)
} }
if !endpointsLastChangeTriggerTime.IsZero() {
newEndpoints.Annotations[v1.EndpointsLastChangeTriggerTime] =
endpointsLastChangeTriggerTime.Format(time.RFC3339Nano)
}
klog.V(4).Infof("Update endpoints for %v/%v, ready: %d not ready: %d", service.Namespace, service.Name, totalReadyEps, totalNotReadyEps) klog.V(4).Infof("Update endpoints for %v/%v, ready: %d not ready: %d", service.Namespace, service.Name, totalReadyEps, totalNotReadyEps)
if createEndpoints { if createEndpoints {
// No previous endpoints, create them // No previous endpoints, create them
......
...@@ -44,6 +44,9 @@ import ( ...@@ -44,6 +44,9 @@ import (
var alwaysReady = func() bool { return true } var alwaysReady = func() bool { return true }
var neverReady = func() bool { return false } var neverReady = func() bool { return false }
var emptyNodeName string var emptyNodeName string
var triggerTime = time.Date(2018, 01, 01, 0, 0, 0, 0, time.UTC)
var triggerTimeString = triggerTime.Format(time.RFC3339Nano)
var oldTriggerTimeString = triggerTime.Add(-time.Hour).Format(time.RFC3339Nano)
func addPods(store cache.Store, namespace string, nPods int, nPorts int, nNotReady int) { func addPods(store cache.Store, namespace string, nPods int, nPorts int, nNotReady int) {
for i := 0; i < nPods+nNotReady; i++ { for i := 0; i < nPods+nNotReady; i++ {
...@@ -1175,3 +1178,94 @@ func TestDetermineNeededServiceUpdates(t *testing.T) { ...@@ -1175,3 +1178,94 @@ func TestDetermineNeededServiceUpdates(t *testing.T) {
} }
} }
} }
func TestLastTriggerChangeTimeAnnotation(t *testing.T) {
ns := "other"
testServer, endpointsHandler := makeTestServer(t, ns)
defer testServer.Close()
endpoints := newController(testServer.URL)
endpoints.endpointsStore.Add(&v1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Namespace: ns,
ResourceVersion: "1",
},
Subsets: []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{{IP: "6.7.8.9", NodeName: &emptyNodeName}},
Ports: []v1.EndpointPort{{Port: 1000, Protocol: "TCP"}},
}},
})
addPods(endpoints.podStore, ns, 1, 1, 0)
endpoints.serviceStore.Add(&v1.Service{
ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: ns, CreationTimestamp: metav1.NewTime(triggerTime)},
Spec: v1.ServiceSpec{
Selector: map[string]string{},
Ports: []v1.ServicePort{{Port: 80, TargetPort: intstr.FromInt(8080), Protocol: "TCP"}},
},
})
endpoints.syncService(ns + "/foo")
endpointsHandler.ValidateRequestCount(t, 1)
data := runtime.EncodeOrDie(testapi.Default.Codec(), &v1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Namespace: ns,
ResourceVersion: "1",
Annotations: map[string]string{
v1.EndpointsLastChangeTriggerTime: triggerTimeString,
},
},
Subsets: []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{{IP: "1.2.3.4", NodeName: &emptyNodeName, TargetRef: &v1.ObjectReference{Kind: "Pod", Name: "pod0", Namespace: ns}}},
Ports: []v1.EndpointPort{{Port: 8080, Protocol: "TCP"}},
}},
})
endpointsHandler.ValidateRequest(t, testapi.Default.ResourcePath("endpoints", ns, "foo"), "PUT", &data)
}
func TestLastTriggerChangeTimeAnnotation_AnnotationOverridden(t *testing.T) {
ns := "other"
testServer, endpointsHandler := makeTestServer(t, ns)
defer testServer.Close()
endpoints := newController(testServer.URL)
endpoints.endpointsStore.Add(&v1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Namespace: ns,
ResourceVersion: "1",
Annotations: map[string]string{
v1.EndpointsLastChangeTriggerTime: oldTriggerTimeString,
},
},
Subsets: []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{{IP: "6.7.8.9", NodeName: &emptyNodeName}},
Ports: []v1.EndpointPort{{Port: 1000, Protocol: "TCP"}},
}},
})
addPods(endpoints.podStore, ns, 1, 1, 0)
endpoints.serviceStore.Add(&v1.Service{
ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: ns, CreationTimestamp: metav1.NewTime(triggerTime)},
Spec: v1.ServiceSpec{
Selector: map[string]string{},
Ports: []v1.ServicePort{{Port: 80, TargetPort: intstr.FromInt(8080), Protocol: "TCP"}},
},
})
endpoints.syncService(ns + "/foo")
endpointsHandler.ValidateRequestCount(t, 1)
data := runtime.EncodeOrDie(testapi.Default.Codec(), &v1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Namespace: ns,
ResourceVersion: "1",
Annotations: map[string]string{
v1.EndpointsLastChangeTriggerTime: triggerTimeString,
},
},
Subsets: []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{{IP: "1.2.3.4", NodeName: &emptyNodeName, TargetRef: &v1.ObjectReference{Kind: "Pod", Name: "pod0", Namespace: ns}}},
Ports: []v1.EndpointPort{{Port: 8080, Protocol: "TCP"}},
}},
})
endpointsHandler.ValidateRequest(t, testapi.Default.ResourcePath("endpoints", ns, "foo"), "PUT", &data)
}
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package endpoint
import (
"sync"
"time"
"k8s.io/api/core/v1"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
)
// TriggerTimeTracker is a util used to compute the EndpointsLastChangeTriggerTime annotation which
// is exported in the endpoints controller's sync function.
// See the documentation of the EndpointsLastChangeTriggerTime annotation for more details.
//
// Please note that this util may compute a wrong EndpointsLastChangeTriggerTime if a same object
// changes multiple times between two consecutive syncs. We're aware of this limitation but we
// decided to accept it, as fixing it would require a major rewrite of the endpoints controller and
// Informer framework. Such situations, i.e. frequent updates of the same object in a single sync
// period, should be relatively rare and therefore this util should provide a good approximation of
// the EndpointsLastChangeTriggerTime.
// TODO(mm4tt): Implement a more robust mechanism that is not subject to the above limitations.
type TriggerTimeTracker struct {
// endpointsStates is a map, indexed by Endpoints object key, storing the last known Endpoints
// object state observed during the most recent call of the ComputeEndpointsLastChangeTriggerTime
// function.
endpointsStates map[endpointsKey]endpointsState
// mutex guarding the endpointsStates map.
mutex sync.Mutex
}
// NewTriggerTimeTracker creates a new instance of the TriggerTimeTracker.
func NewTriggerTimeTracker() *TriggerTimeTracker {
return &TriggerTimeTracker{
endpointsStates: make(map[endpointsKey]endpointsState),
}
}
// endpointsKey is a key uniquely identifying an Endpoints object.
type endpointsKey struct {
// namespace, name composing a namespaced name - an unique identifier of every Endpoints object.
namespace, name string
}
// endpointsState represents a state of an Endpoints object that is known to this util.
type endpointsState struct {
// lastServiceTriggerTime is a service trigger time observed most recently.
lastServiceTriggerTime time.Time
// lastPodTriggerTimes is a map (Pod name -> time) storing the pod trigger times that were
// observed during the most recent call of the ComputeEndpointsLastChangeTriggerTime function.
lastPodTriggerTimes map[string]time.Time
}
// ComputeEndpointsLastChangeTriggerTime updates the state of the Endpoints object being synced
// and returns the time that should be exported as the EndpointsLastChangeTriggerTime annotation.
//
// If the method returns a 'zero' time the EndpointsLastChangeTriggerTime annotation shouldn't be
// exported.
//
// Please note that this function may compute a wrong EndpointsLastChangeTriggerTime value if the
// same object (pod/service) changes multiple times between two consecutive syncs.
//
// Important: This method is go-routing safe but only when called for different keys. The method
// shouldn't be called concurrently for the same key! This contract is fulfilled in the current
// implementation of the endpoints controller.
func (t *TriggerTimeTracker) ComputeEndpointsLastChangeTriggerTime(
namespace, name string, service *v1.Service, pods []*v1.Pod) time.Time {
key := endpointsKey{namespace: namespace, name: name}
// As there won't be any concurrent calls for the same key, we need to guard access only to the
// endpointsStates map.
t.mutex.Lock()
state, wasKnown := t.endpointsStates[key]
t.mutex.Unlock()
// Update the state before returning.
defer func() {
t.mutex.Lock()
t.endpointsStates[key] = state
t.mutex.Unlock()
}()
// minChangedTriggerTime is the min trigger time of all trigger times that have changed since the
// last sync.
var minChangedTriggerTime time.Time
// TODO(mm4tt): If memory allocation / GC performance impact of recreating map in every call
// turns out to be too expensive, we should consider rewriting this to reuse the existing map.
podTriggerTimes := make(map[string]time.Time)
for _, pod := range pods {
if podTriggerTime := getPodTriggerTime(pod); !podTriggerTime.IsZero() {
podTriggerTimes[pod.Name] = podTriggerTime
if podTriggerTime.After(state.lastPodTriggerTimes[pod.Name]) {
// Pod trigger time has changed since the last sync, update minChangedTriggerTime.
minChangedTriggerTime = min(minChangedTriggerTime, podTriggerTime)
}
}
}
serviceTriggerTime := getServiceTriggerTime(service)
if serviceTriggerTime.After(state.lastServiceTriggerTime) {
// Service trigger time has changed since the last sync, update minChangedTriggerTime.
minChangedTriggerTime = min(minChangedTriggerTime, serviceTriggerTime)
}
state.lastPodTriggerTimes = podTriggerTimes
state.lastServiceTriggerTime = serviceTriggerTime
if !wasKnown {
// New Endpoints object / new Service, use Service creationTimestamp.
return service.CreationTimestamp.Time
} else {
// Regular update of the Endpoints object, return min of changed trigger times.
return minChangedTriggerTime
}
}
// DeleteEndpoints deletes endpoints state stored in this util.
func (t *TriggerTimeTracker) DeleteEndpoints(namespace, name string) {
key := endpointsKey{namespace: namespace, name: name}
t.mutex.Lock()
defer t.mutex.Unlock()
delete(t.endpointsStates, key)
}
// getPodTriggerTime returns the time of the pod change (trigger) that resulted or will result in
// the endpoints object change.
func getPodTriggerTime(pod *v1.Pod) (triggerTime time.Time) {
if readyCondition := podutil.GetPodReadyCondition(pod.Status); readyCondition != nil {
triggerTime = readyCondition.LastTransitionTime.Time
}
// TODO(mm4tt): Implement missing cases: deletionTime set, pod label change
return triggerTime
}
// getServiceTriggerTime returns the time of the service change (trigger) that resulted or will
// result in the endpoints object change.
func getServiceTriggerTime(service *v1.Service) (triggerTime time.Time) {
// TODO(mm4tt): Ideally we should look at service.LastUpdateTime, but such thing doesn't exist.
return service.CreationTimestamp.Time
}
// min returns minimum of the currentMin and newValue or newValue if the currentMin is not set.
func min(currentMin, newValue time.Time) time.Time {
if currentMin.IsZero() || newValue.Before(currentMin) {
return newValue
}
return currentMin
}
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package endpoint
import (
"runtime"
"testing"
"time"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
var (
t0 = time.Date(2019, 01, 01, 0, 0, 0, 0, time.UTC)
t1 = t0.Add(time.Second)
t2 = t1.Add(time.Second)
t3 = t2.Add(time.Second)
t4 = t3.Add(time.Second)
t5 = t4.Add(time.Second)
ns = "ns1"
name = "my-service"
)
func TestNewService_NoPods(t *testing.T) {
tester := newTester(t)
service := createService(ns, name, t2)
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service).expect(t2)
}
func TestNewService_ExistingPods(t *testing.T) {
tester := newTester(t)
service := createService(ns, name, t3)
pod1 := createPod(ns, "pod1", t0)
pod2 := createPod(ns, "pod2", t1)
pod3 := createPod(ns, "pod3", t5)
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1, pod2, pod3).
// Pods were created before service, but trigger time is the time when service was created.
expect(t3)
}
func TestPodsAdded(t *testing.T) {
tester := newTester(t)
service := createService(ns, name, t0)
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service).expect(t0)
pod1 := createPod(ns, "pod1", t2)
pod2 := createPod(ns, "pod2", t1)
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1, pod2).expect(t1)
}
func TestPodsUpdated(t *testing.T) {
tester := newTester(t)
service := createService(ns, name, t0)
pod1 := createPod(ns, "pod1", t1)
pod2 := createPod(ns, "pod2", t2)
pod3 := createPod(ns, "pod3", t3)
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1, pod2, pod3).expect(t0)
pod1 = createPod(ns, "pod1", t5)
pod2 = createPod(ns, "pod2", t4)
// pod3 doesn't change.
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1, pod2, pod3).expect(t4)
}
func TestPodsUpdated_NoOp(t *testing.T) {
tester := newTester(t)
service := createService(ns, name, t0)
pod1 := createPod(ns, "pod1", t1)
pod2 := createPod(ns, "pod2", t2)
pod3 := createPod(ns, "pod3", t3)
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1, pod2, pod3).expect(t0)
// Nothing has changed.
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1, pod2, pod3).expectNil()
}
func TestPodDeletedThenAdded(t *testing.T) {
tester := newTester(t)
service := createService(ns, name, t0)
pod1 := createPod(ns, "pod1", t1)
pod2 := createPod(ns, "pod2", t2)
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1, pod2).expect(t0)
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1).expectNil()
pod2 = createPod(ns, "pod2", t4)
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1, pod2).expect(t4)
}
func TestServiceDeletedThenAdded(t *testing.T) {
tester := newTester(t)
service := createService(ns, name, t0)
pod1 := createPod(ns, "pod1", t1)
pod2 := createPod(ns, "pod2", t2)
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1, pod2).expect(t0)
tester.DeleteEndpoints(ns, name)
service = createService(ns, name, t3)
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1, pod2).expect(t3)
}
func TestServiceUpdated_NoPodChange(t *testing.T) {
tester := newTester(t)
service := createService(ns, name, t0)
pod1 := createPod(ns, "pod1", t1)
pod2 := createPod(ns, "pod2", t2)
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1, pod2).expect(t0)
// service's ports have changed.
service.Spec = v1.ServiceSpec{
Selector: map[string]string{},
Ports: []v1.ServicePort{{Port: 80, TargetPort: intstr.FromInt(8080), Protocol: "TCP"}},
}
// Currently we're not able to calculate trigger time for service updates, hence the returned
// value is a nil time.
tester.whenComputeEndpointsLastChangeTriggerTime(ns, name, service, pod1, pod2).expectNil()
}
// ------- Test Utils -------
type tester struct {
*TriggerTimeTracker
t *testing.T
}
func newTester(t *testing.T) *tester {
return &tester{NewTriggerTimeTracker(), t}
}
func (t *tester) whenComputeEndpointsLastChangeTriggerTime(
namespace, name string, service *v1.Service, pods ...*v1.Pod) subject {
return subject{t.ComputeEndpointsLastChangeTriggerTime(namespace, name, service, pods), t.t}
}
type subject struct {
got time.Time
t *testing.T
}
func (s subject) expect(expected time.Time) {
s.doExpect(expected)
}
func (s subject) expectNil() {
s.doExpect(time.Time{})
}
func (s subject) doExpect(expected time.Time) {
if s.got != expected {
_, fn, line, _ := runtime.Caller(2)
s.t.Errorf("Wrong trigger time in %s:%d expected %s, got %s", fn, line, expected, s.got)
}
}
func createPod(namespace, name string, readyTime time.Time) *v1.Pod {
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
Status: v1.PodStatus{Conditions: []v1.PodCondition{
{
Type: v1.PodReady,
Status: v1.ConditionTrue,
LastTransitionTime: metav1.NewTime(readyTime),
},
},
},
}
}
func createService(namespace, name string, creationTime time.Time) *v1.Service {
return &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: name,
CreationTimestamp: metav1.NewTime(creationTime),
},
}
}
...@@ -17,6 +17,7 @@ go_library( ...@@ -17,6 +17,7 @@ go_library(
"scheduler_binder.go", "scheduler_binder.go",
"scheduler_binder_cache.go", "scheduler_binder_cache.go",
"scheduler_binder_fake.go", "scheduler_binder_fake.go",
"util.go",
"volume_host.go", "volume_host.go",
], ],
importpath = "k8s.io/kubernetes/pkg/controller/volume/persistentvolume", importpath = "k8s.io/kubernetes/pkg/controller/volume/persistentvolume",
......
...@@ -185,7 +185,7 @@ func findMatchingVolume( ...@@ -185,7 +185,7 @@ func findMatchingVolume(
} }
} }
if isVolumeBoundToClaim(volume, claim) { if IsVolumeBoundToClaim(volume, claim) {
// this claim and volume are pre-bound; return // this claim and volume are pre-bound; return
// the volume if the size request is satisfied, // the volume if the size request is satisfied,
// otherwise continue searching for a match // otherwise continue searching for a match
......
...@@ -294,24 +294,6 @@ func (ctrl *PersistentVolumeController) isDelayBindingProvisioning(claim *v1.Per ...@@ -294,24 +294,6 @@ func (ctrl *PersistentVolumeController) isDelayBindingProvisioning(claim *v1.Per
return ok return ok
} }
func (ctrl *PersistentVolumeController) isDelayBindingMode(claim *v1.PersistentVolumeClaim) (bool, error) {
className := v1helper.GetPersistentVolumeClaimClass(claim)
if className == "" {
return false, nil
}
class, err := ctrl.classLister.Get(className)
if err != nil {
return false, nil
}
if class.VolumeBindingMode == nil {
return false, fmt.Errorf("VolumeBindingMode not set for StorageClass %q", className)
}
return *class.VolumeBindingMode == storage.VolumeBindingWaitForFirstConsumer, nil
}
// shouldDelayBinding returns true if binding of claim should be delayed, false otherwise. // shouldDelayBinding returns true if binding of claim should be delayed, false otherwise.
// If binding of claim should be delayed, only claims pbound by scheduler // If binding of claim should be delayed, only claims pbound by scheduler
func (ctrl *PersistentVolumeController) shouldDelayBinding(claim *v1.PersistentVolumeClaim) (bool, error) { func (ctrl *PersistentVolumeController) shouldDelayBinding(claim *v1.PersistentVolumeClaim) (bool, error) {
...@@ -321,7 +303,7 @@ func (ctrl *PersistentVolumeController) shouldDelayBinding(claim *v1.PersistentV ...@@ -321,7 +303,7 @@ func (ctrl *PersistentVolumeController) shouldDelayBinding(claim *v1.PersistentV
} }
// If claim is in delay binding mode. // If claim is in delay binding mode.
return ctrl.isDelayBindingMode(claim) return IsDelayBindingMode(claim, ctrl.classLister)
} }
// syncUnboundClaim is the main controller method to decide what to do with an // syncUnboundClaim is the main controller method to decide what to do with an
...@@ -419,7 +401,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol ...@@ -419,7 +401,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol
} }
// OBSERVATION: pvc is "Bound", pv is "Bound" // OBSERVATION: pvc is "Bound", pv is "Bound"
return nil return nil
} else if isVolumeBoundToClaim(volume, claim) { } else if IsVolumeBoundToClaim(volume, claim) {
// User asked for a PV that is claimed by this PVC // User asked for a PV that is claimed by this PVC
// OBSERVATION: pvc is "Pending", pv is "Bound" // OBSERVATION: pvc is "Pending", pv is "Bound"
klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume already bound, finishing the binding", claimToClaimKey(claim)) klog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: volume already bound, finishing the binding", claimToClaimKey(claim))
...@@ -863,7 +845,7 @@ func (ctrl *PersistentVolumeController) updateVolumePhaseWithEvent(volume *v1.Pe ...@@ -863,7 +845,7 @@ func (ctrl *PersistentVolumeController) updateVolumePhaseWithEvent(volume *v1.Pe
func (ctrl *PersistentVolumeController) bindVolumeToClaim(volume *v1.PersistentVolume, claim *v1.PersistentVolumeClaim) (*v1.PersistentVolume, error) { func (ctrl *PersistentVolumeController) bindVolumeToClaim(volume *v1.PersistentVolume, claim *v1.PersistentVolumeClaim) (*v1.PersistentVolume, error) {
klog.V(4).Infof("updating PersistentVolume[%s]: binding to %q", volume.Name, claimToClaimKey(claim)) klog.V(4).Infof("updating PersistentVolume[%s]: binding to %q", volume.Name, claimToClaimKey(claim))
volumeClone, dirty, err := ctrl.getBindVolumeToClaim(volume, claim) volumeClone, dirty, err := GetBindVolumeToClaim(volume, claim)
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -897,43 +879,6 @@ func (ctrl *PersistentVolumeController) updateBindVolumeToClaim(volumeClone *v1. ...@@ -897,43 +879,6 @@ func (ctrl *PersistentVolumeController) updateBindVolumeToClaim(volumeClone *v1.
return newVol, nil return newVol, nil
} }
// Get new PV object only, no API or cache update
func (ctrl *PersistentVolumeController) getBindVolumeToClaim(volume *v1.PersistentVolume, claim *v1.PersistentVolumeClaim) (*v1.PersistentVolume, bool, error) {
dirty := false
// Check if the volume was already bound (either by user or by controller)
shouldSetBoundByController := false
if !isVolumeBoundToClaim(volume, claim) {
shouldSetBoundByController = true
}
// The volume from method args can be pointing to watcher cache. We must not
// modify these, therefore create a copy.
volumeClone := volume.DeepCopy()
// Bind the volume to the claim if it is not bound yet
if volume.Spec.ClaimRef == nil ||
volume.Spec.ClaimRef.Name != claim.Name ||
volume.Spec.ClaimRef.Namespace != claim.Namespace ||
volume.Spec.ClaimRef.UID != claim.UID {
claimRef, err := ref.GetReference(scheme.Scheme, claim)
if err != nil {
return nil, false, fmt.Errorf("Unexpected error getting claim reference: %v", err)
}
volumeClone.Spec.ClaimRef = claimRef
dirty = true
}
// Set annBoundByController if it is not set yet
if shouldSetBoundByController && !metav1.HasAnnotation(volumeClone.ObjectMeta, annBoundByController) {
metav1.SetMetaDataAnnotation(&volumeClone.ObjectMeta, annBoundByController, "yes")
dirty = true
}
return volumeClone, dirty, nil
}
// bindClaimToVolume modifies the given claim to be bound to a volume and // bindClaimToVolume modifies the given claim to be bound to a volume and
// saves it to API server. The volume is not modified in this method! // saves it to API server. The volume is not modified in this method!
func (ctrl *PersistentVolumeController) bindClaimToVolume(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) (*v1.PersistentVolumeClaim, error) { func (ctrl *PersistentVolumeController) bindClaimToVolume(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) (*v1.PersistentVolumeClaim, error) {
......
...@@ -467,22 +467,6 @@ func getVolumeStatusForLogging(volume *v1.PersistentVolume) string { ...@@ -467,22 +467,6 @@ func getVolumeStatusForLogging(volume *v1.PersistentVolume) string {
return fmt.Sprintf("phase: %s, bound to: %q, boundByController: %v", volume.Status.Phase, claimName, boundByController) return fmt.Sprintf("phase: %s, bound to: %q, boundByController: %v", volume.Status.Phase, claimName, boundByController)
} }
// isVolumeBoundToClaim returns true, if given volume is pre-bound or bound
// to specific claim. Both claim.Name and claim.Namespace must be equal.
// If claim.UID is present in volume.Spec.ClaimRef, it must be equal too.
func isVolumeBoundToClaim(volume *v1.PersistentVolume, claim *v1.PersistentVolumeClaim) bool {
if volume.Spec.ClaimRef == nil {
return false
}
if claim.Name != volume.Spec.ClaimRef.Name || claim.Namespace != volume.Spec.ClaimRef.Namespace {
return false
}
if volume.Spec.ClaimRef.UID != "" && claim.UID != volume.Spec.ClaimRef.UID {
return false
}
return true
}
// storeObjectUpdate updates given cache with a new object version from Informer // storeObjectUpdate updates given cache with a new object version from Informer
// callback (i.e. with events from etcd) or with an object modified by the // callback (i.e. with events from etcd) or with an object modified by the
// controller itself. Returns "true", if the cache was updated, false if the // controller itself. Returns "true", if the cache was updated, false if the
......
...@@ -29,6 +29,7 @@ import ( ...@@ -29,6 +29,7 @@ import (
coreinformers "k8s.io/client-go/informers/core/v1" coreinformers "k8s.io/client-go/informers/core/v1"
storageinformers "k8s.io/client-go/informers/storage/v1" storageinformers "k8s.io/client-go/informers/storage/v1"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
storagelisters "k8s.io/client-go/listers/storage/v1"
"k8s.io/klog" "k8s.io/klog"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
volumeutil "k8s.io/kubernetes/pkg/volume/util" volumeutil "k8s.io/kubernetes/pkg/volume/util"
...@@ -97,7 +98,8 @@ type SchedulerVolumeBinder interface { ...@@ -97,7 +98,8 @@ type SchedulerVolumeBinder interface {
} }
type volumeBinder struct { type volumeBinder struct {
ctrl *PersistentVolumeController kubeClient clientset.Interface
classLister storagelisters.StorageClassLister
nodeInformer coreinformers.NodeInformer nodeInformer coreinformers.NodeInformer
pvcCache PVCAssumeCache pvcCache PVCAssumeCache
...@@ -120,14 +122,9 @@ func NewVolumeBinder( ...@@ -120,14 +122,9 @@ func NewVolumeBinder(
storageClassInformer storageinformers.StorageClassInformer, storageClassInformer storageinformers.StorageClassInformer,
bindTimeout time.Duration) SchedulerVolumeBinder { bindTimeout time.Duration) SchedulerVolumeBinder {
// TODO: find better way...
ctrl := &PersistentVolumeController{
kubeClient: kubeClient,
classLister: storageClassInformer.Lister(),
}
b := &volumeBinder{ b := &volumeBinder{
ctrl: ctrl, kubeClient: kubeClient,
classLister: storageClassInformer.Lister(),
nodeInformer: nodeInformer, nodeInformer: nodeInformer,
pvcCache: NewPVCAssumeCache(pvcInformer.Informer()), pvcCache: NewPVCAssumeCache(pvcInformer.Informer()),
pvCache: NewPVAssumeCache(pvInformer.Informer()), pvCache: NewPVAssumeCache(pvInformer.Informer()),
...@@ -291,8 +288,8 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string) (al ...@@ -291,8 +288,8 @@ func (b *volumeBinder) AssumePodVolumes(assumedPod *v1.Pod, nodeName string) (al
// Assume PV // Assume PV
newBindings := []*bindingInfo{} newBindings := []*bindingInfo{}
for _, binding := range claimsToBind { for _, binding := range claimsToBind {
newPV, dirty, err := b.ctrl.getBindVolumeToClaim(binding.pv, binding.pvc) newPV, dirty, err := GetBindVolumeToClaim(binding.pv, binding.pvc)
klog.V(5).Infof("AssumePodVolumes: getBindVolumeToClaim for pod %q, PV %q, PVC %q. newPV %p, dirty %v, err: %v", klog.V(5).Infof("AssumePodVolumes: GetBindVolumeToClaim for pod %q, PV %q, PVC %q. newPV %p, dirty %v, err: %v",
podName, podName,
binding.pv.Name, binding.pv.Name,
binding.pvc.Name, binding.pvc.Name,
...@@ -411,9 +408,13 @@ func (b *volumeBinder) bindAPIUpdate(podName string, bindings []*bindingInfo, cl ...@@ -411,9 +408,13 @@ func (b *volumeBinder) bindAPIUpdate(podName string, bindings []*bindingInfo, cl
for _, binding = range bindings { for _, binding = range bindings {
klog.V(5).Infof("bindAPIUpdate: Pod %q, binding PV %q to PVC %q", podName, binding.pv.Name, binding.pvc.Name) klog.V(5).Infof("bindAPIUpdate: Pod %q, binding PV %q to PVC %q", podName, binding.pv.Name, binding.pvc.Name)
// TODO: does it hurt if we make an api call and nothing needs to be updated? // TODO: does it hurt if we make an api call and nothing needs to be updated?
if newPV, err := b.ctrl.updateBindVolumeToClaim(binding.pv, binding.pvc, false); err != nil { claimKey := claimToClaimKey(binding.pvc)
klog.V(2).Infof("claim %q bound to volume %q", claimKey, binding.pv.Name)
if newPV, err := b.kubeClient.CoreV1().PersistentVolumes().Update(binding.pv); err != nil {
klog.V(4).Infof("updating PersistentVolume[%s]: binding to %q failed: %v", binding.pv.Name, claimKey, err)
return err return err
} else { } else {
klog.V(4).Infof("updating PersistentVolume[%s]: bound to %q", binding.pv.Name, claimKey)
// Save updated object from apiserver for later checking. // Save updated object from apiserver for later checking.
binding.pv = newPV binding.pv = newPV
} }
...@@ -424,7 +425,7 @@ func (b *volumeBinder) bindAPIUpdate(podName string, bindings []*bindingInfo, cl ...@@ -424,7 +425,7 @@ func (b *volumeBinder) bindAPIUpdate(podName string, bindings []*bindingInfo, cl
// PV controller is expect to signal back by removing related annotations if actual provisioning fails // PV controller is expect to signal back by removing related annotations if actual provisioning fails
for i, claim = range claimsToProvision { for i, claim = range claimsToProvision {
klog.V(5).Infof("bindAPIUpdate: Pod %q, PVC %q", podName, getPVCName(claim)) klog.V(5).Infof("bindAPIUpdate: Pod %q, PVC %q", podName, getPVCName(claim))
if newClaim, err := b.ctrl.kubeClient.CoreV1().PersistentVolumeClaims(claim.Namespace).Update(claim); err != nil { if newClaim, err := b.kubeClient.CoreV1().PersistentVolumeClaims(claim.Namespace).Update(claim); err != nil {
return err return err
} else { } else {
// Save updated object from apiserver for later checking. // Save updated object from apiserver for later checking.
...@@ -627,7 +628,7 @@ func (b *volumeBinder) getPodVolumes(pod *v1.Pod) (boundClaims []*v1.PersistentV ...@@ -627,7 +628,7 @@ func (b *volumeBinder) getPodVolumes(pod *v1.Pod) (boundClaims []*v1.PersistentV
if volumeBound { if volumeBound {
boundClaims = append(boundClaims, pvc) boundClaims = append(boundClaims, pvc)
} else { } else {
delayBindingMode, err := b.ctrl.isDelayBindingMode(pvc) delayBindingMode, err := IsDelayBindingMode(pvc, b.classLister)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, nil, err
} }
...@@ -726,7 +727,7 @@ func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v ...@@ -726,7 +727,7 @@ func (b *volumeBinder) checkVolumeProvisions(pod *v1.Pod, claimsToProvision []*v
return false, nil, fmt.Errorf("no class for claim %q", pvcName) return false, nil, fmt.Errorf("no class for claim %q", pvcName)
} }
class, err := b.ctrl.classLister.Get(className) class, err := b.classLister.Get(className)
if err != nil { if err != nil {
return false, nil, fmt.Errorf("failed to find storage class %q", className) return false, nil, fmt.Errorf("failed to find storage class %q", className)
} }
......
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package persistentvolume
import (
"fmt"
"k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/scheme"
storagelisters "k8s.io/client-go/listers/storage/v1"
"k8s.io/client-go/tools/reference"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
)
// IsDelayBindingMode checks if claim is in delay binding mode.
func IsDelayBindingMode(claim *v1.PersistentVolumeClaim, classLister storagelisters.StorageClassLister) (bool, error) {
className := v1helper.GetPersistentVolumeClaimClass(claim)
if className == "" {
return false, nil
}
class, err := classLister.Get(className)
if err != nil {
return false, nil
}
if class.VolumeBindingMode == nil {
return false, fmt.Errorf("VolumeBindingMode not set for StorageClass %q", className)
}
return *class.VolumeBindingMode == storage.VolumeBindingWaitForFirstConsumer, nil
}
// GetBindVolumeToClaim returns a new volume which is bound to given claim. In
// addition, it returns a bool which indicates whether we made modification on
// original volume.
func GetBindVolumeToClaim(volume *v1.PersistentVolume, claim *v1.PersistentVolumeClaim) (*v1.PersistentVolume, bool, error) {
dirty := false
// Check if the volume was already bound (either by user or by controller)
shouldSetBoundByController := false
if !IsVolumeBoundToClaim(volume, claim) {
shouldSetBoundByController = true
}
// The volume from method args can be pointing to watcher cache. We must not
// modify these, therefore create a copy.
volumeClone := volume.DeepCopy()
// Bind the volume to the claim if it is not bound yet
if volume.Spec.ClaimRef == nil ||
volume.Spec.ClaimRef.Name != claim.Name ||
volume.Spec.ClaimRef.Namespace != claim.Namespace ||
volume.Spec.ClaimRef.UID != claim.UID {
claimRef, err := reference.GetReference(scheme.Scheme, claim)
if err != nil {
return nil, false, fmt.Errorf("Unexpected error getting claim reference: %v", err)
}
volumeClone.Spec.ClaimRef = claimRef
dirty = true
}
// Set annBoundByController if it is not set yet
if shouldSetBoundByController && !metav1.HasAnnotation(volumeClone.ObjectMeta, annBoundByController) {
metav1.SetMetaDataAnnotation(&volumeClone.ObjectMeta, annBoundByController, "yes")
dirty = true
}
return volumeClone, dirty, nil
}
// IsVolumeBoundToClaim returns true, if given volume is pre-bound or bound
// to specific claim. Both claim.Name and claim.Namespace must be equal.
// If claim.UID is present in volume.Spec.ClaimRef, it must be equal too.
func IsVolumeBoundToClaim(volume *v1.PersistentVolume, claim *v1.PersistentVolumeClaim) bool {
if volume.Spec.ClaimRef == nil {
return false
}
if claim.Name != volume.Spec.ClaimRef.Name || claim.Namespace != volume.Spec.ClaimRef.Namespace {
return false
}
if volume.Spec.ClaimRef.UID != "" && claim.UID != volume.Spec.ClaimRef.UID {
return false
}
return true
}
...@@ -93,14 +93,15 @@ var ( ...@@ -93,14 +93,15 @@ var (
type DeleteOptions struct { type DeleteOptions struct {
resource.FilenameOptions resource.FilenameOptions
LabelSelector string LabelSelector string
FieldSelector string FieldSelector string
DeleteAll bool DeleteAll bool
IgnoreNotFound bool DeleteAllNamespaces bool
Cascade bool IgnoreNotFound bool
DeleteNow bool Cascade bool
ForceDeletion bool DeleteNow bool
WaitForDeletion bool ForceDeletion bool
WaitForDeletion bool
GracePeriod int GracePeriod int
Timeout time.Duration Timeout time.Duration
...@@ -170,6 +171,7 @@ func (o *DeleteOptions) Complete(f cmdutil.Factory, args []string, cmd *cobra.Co ...@@ -170,6 +171,7 @@ func (o *DeleteOptions) Complete(f cmdutil.Factory, args []string, cmd *cobra.Co
LabelSelectorParam(o.LabelSelector). LabelSelectorParam(o.LabelSelector).
FieldSelectorParam(o.FieldSelector). FieldSelectorParam(o.FieldSelector).
SelectAllParam(o.DeleteAll). SelectAllParam(o.DeleteAll).
AllNamespaces(o.DeleteAllNamespaces).
ResourceTypeOrNameArgs(false, args...).RequireObject(false). ResourceTypeOrNameArgs(false, args...).RequireObject(false).
Flatten(). Flatten().
Do() Do()
......
...@@ -33,6 +33,7 @@ type DeleteFlags struct { ...@@ -33,6 +33,7 @@ type DeleteFlags struct {
FieldSelector *string FieldSelector *string
All *bool All *bool
AllNamespaces *bool
Cascade *bool Cascade *bool
Force *bool Force *bool
GracePeriod *int GracePeriod *int
...@@ -68,6 +69,9 @@ func (f *DeleteFlags) ToOptions(dynamicClient dynamic.Interface, streams generic ...@@ -68,6 +69,9 @@ func (f *DeleteFlags) ToOptions(dynamicClient dynamic.Interface, streams generic
if f.All != nil { if f.All != nil {
options.DeleteAll = *f.All options.DeleteAll = *f.All
} }
if f.AllNamespaces != nil {
options.DeleteAllNamespaces = *f.AllNamespaces
}
if f.Cascade != nil { if f.Cascade != nil {
options.Cascade = *f.Cascade options.Cascade = *f.Cascade
} }
...@@ -104,6 +108,9 @@ func (f *DeleteFlags) AddFlags(cmd *cobra.Command) { ...@@ -104,6 +108,9 @@ func (f *DeleteFlags) AddFlags(cmd *cobra.Command) {
if f.All != nil { if f.All != nil {
cmd.Flags().BoolVar(f.All, "all", *f.All, "Delete all resources, including uninitialized ones, in the namespace of the specified resource types.") cmd.Flags().BoolVar(f.All, "all", *f.All, "Delete all resources, including uninitialized ones, in the namespace of the specified resource types.")
} }
if f.AllNamespaces != nil {
cmd.Flags().BoolVarP(f.AllNamespaces, "all-namespaces", "A", *f.AllNamespaces, "If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.")
}
if f.Force != nil { if f.Force != nil {
cmd.Flags().BoolVar(f.Force, "force", *f.Force, "Only used when grace-period=0. If true, immediately remove resources from API and bypass graceful deletion. Note that immediate deletion of some resources may result in inconsistency or data loss and requires confirmation.") cmd.Flags().BoolVar(f.Force, "force", *f.Force, "Only used when grace-period=0. If true, immediately remove resources from API and bypass graceful deletion. Note that immediate deletion of some resources may result in inconsistency or data loss and requires confirmation.")
} }
...@@ -137,6 +144,7 @@ func NewDeleteCommandFlags(usage string) *DeleteFlags { ...@@ -137,6 +144,7 @@ func NewDeleteCommandFlags(usage string) *DeleteFlags {
// setup command defaults // setup command defaults
all := false all := false
allNamespaces := false
force := false force := false
ignoreNotFound := false ignoreNotFound := false
now := false now := false
...@@ -158,6 +166,7 @@ func NewDeleteCommandFlags(usage string) *DeleteFlags { ...@@ -158,6 +166,7 @@ func NewDeleteCommandFlags(usage string) *DeleteFlags {
GracePeriod: &gracePeriod, GracePeriod: &gracePeriod,
All: &all, All: &all,
AllNamespaces: &allNamespaces,
Force: &force, Force: &force,
IgnoreNotFound: &ignoreNotFound, IgnoreNotFound: &ignoreNotFound,
Now: &now, Now: &now,
......
...@@ -23,6 +23,7 @@ go_library( ...@@ -23,6 +23,7 @@ go_library(
"//vendor/k8s.io/kube-openapi/pkg/util/proto:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/util/proto:go_default_library",
"//vendor/sigs.k8s.io/structured-merge-diff/fieldpath:go_default_library", "//vendor/sigs.k8s.io/structured-merge-diff/fieldpath:go_default_library",
"//vendor/sigs.k8s.io/structured-merge-diff/merge:go_default_library", "//vendor/sigs.k8s.io/structured-merge-diff/merge:go_default_library",
"//vendor/sigs.k8s.io/structured-merge-diff/schema:go_default_library",
"//vendor/sigs.k8s.io/structured-merge-diff/typed:go_default_library", "//vendor/sigs.k8s.io/structured-merge-diff/typed:go_default_library",
"//vendor/sigs.k8s.io/structured-merge-diff/value:go_default_library", "//vendor/sigs.k8s.io/structured-merge-diff/value:go_default_library",
"//vendor/sigs.k8s.io/yaml:go_default_library", "//vendor/sigs.k8s.io/yaml:go_default_library",
......
...@@ -22,6 +22,7 @@ import ( ...@@ -22,6 +22,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kube-openapi/pkg/schemaconv" "k8s.io/kube-openapi/pkg/schemaconv"
"k8s.io/kube-openapi/pkg/util/proto" "k8s.io/kube-openapi/pkg/util/proto"
smdschema "sigs.k8s.io/structured-merge-diff/schema"
"sigs.k8s.io/structured-merge-diff/typed" "sigs.k8s.io/structured-merge-diff/typed"
) )
...@@ -48,6 +49,7 @@ func newGVKParser(models proto.Models) (*gvkParser, error) { ...@@ -48,6 +49,7 @@ func newGVKParser(models proto.Models) (*gvkParser, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to convert models to schema: %v", err) return nil, fmt.Errorf("failed to convert models to schema: %v", err)
} }
typeSchema = makeRawExtensionUntyped(typeSchema)
parser := gvkParser{ parser := gvkParser{
gvks: map[schema.GroupVersionKind]string{}, gvks: map[schema.GroupVersionKind]string{},
} }
...@@ -114,3 +116,20 @@ func parseGroupVersionKind(s proto.Schema) []schema.GroupVersionKind { ...@@ -114,3 +116,20 @@ func parseGroupVersionKind(s proto.Schema) []schema.GroupVersionKind {
return gvkListResult return gvkListResult
} }
// makeRawExtensionUntyped explicitly sets RawExtension's type in the schema to Untyped atomic
// TODO: remove this once kube-openapi is updated to include
// https://github.com/kubernetes/kube-openapi/pull/133
func makeRawExtensionUntyped(s *smdschema.Schema) *smdschema.Schema {
s2 := &smdschema.Schema{}
for _, t := range s.Types {
t2 := t
if t2.Name == "io.k8s.apimachinery.pkg.runtime.RawExtension" {
t2.Atom = smdschema.Atom{
Untyped: &smdschema.Untyped{},
}
}
s2.Types = append(s2.Types, t2)
}
return s2
}
...@@ -112,11 +112,47 @@ ...@@ -112,11 +112,47 @@
}, },
{ {
"ImportPath": "golang.org/x/tools/go/ast/astutil", "ImportPath": "golang.org/x/tools/go/ast/astutil",
"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32" "Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/go/gcexportdata",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/go/internal/cgo",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/go/internal/gcimporter",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/go/internal/packagesdriver",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/go/packages",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
}, },
{ {
"ImportPath": "golang.org/x/tools/imports", "ImportPath": "golang.org/x/tools/imports",
"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32" "Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/internal/fastwalk",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/internal/gopathwalk",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/internal/module",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
},
{
"ImportPath": "golang.org/x/tools/internal/semver",
"Rev": "7f7074d5bcfd282eb16bc382b0bb3da762461985"
}, },
{ {
"ImportPath": "k8s.io/gengo/args", "ImportPath": "k8s.io/gengo/args",
......
...@@ -40,6 +40,11 @@ func NewAWSElasticBlockStoreCSITranslator() InTreePlugin { ...@@ -40,6 +40,11 @@ func NewAWSElasticBlockStoreCSITranslator() InTreePlugin {
return &awsElasticBlockStoreCSITranslator{} return &awsElasticBlockStoreCSITranslator{}
} }
// TranslateInTreeStorageClassParametersToCSI translates InTree EBS storage class parameters to CSI storage class
func (t *awsElasticBlockStoreCSITranslator) TranslateInTreeStorageClassParametersToCSI(scParameters map[string]string) (map[string]string, error) {
return scParameters, nil
}
// TranslateInTreePVToCSI takes a PV with AWSElasticBlockStore set from in-tree // TranslateInTreePVToCSI takes a PV with AWSElasticBlockStore set from in-tree
// and converts the AWSElasticBlockStore source to a CSIPersistentVolumeSource // and converts the AWSElasticBlockStore source to a CSIPersistentVolumeSource
func (t *awsElasticBlockStoreCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) { func (t *awsElasticBlockStoreCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
......
...@@ -58,6 +58,11 @@ func NewGCEPersistentDiskCSITranslator() InTreePlugin { ...@@ -58,6 +58,11 @@ func NewGCEPersistentDiskCSITranslator() InTreePlugin {
return &gcePersistentDiskCSITranslator{} return &gcePersistentDiskCSITranslator{}
} }
// TranslateInTreeStorageClassParametersToCSI translates InTree GCE storage class parameters to CSI storage class
func (g *gcePersistentDiskCSITranslator) TranslateInTreeStorageClassParametersToCSI(scParameters map[string]string) (map[string]string, error) {
return scParameters, nil
}
// TranslateInTreePVToCSI takes a PV with GCEPersistentDisk set from in-tree // TranslateInTreePVToCSI takes a PV with GCEPersistentDisk set from in-tree
// and converts the GCEPersistentDisk source to a CSIPersistentVolumeSource // and converts the GCEPersistentDisk source to a CSIPersistentVolumeSource
func (g *gcePersistentDiskCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) { func (g *gcePersistentDiskCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
......
...@@ -20,6 +20,11 @@ import "k8s.io/api/core/v1" ...@@ -20,6 +20,11 @@ import "k8s.io/api/core/v1"
// InTreePlugin handles translations between CSI and in-tree sources in a PV // InTreePlugin handles translations between CSI and in-tree sources in a PV
type InTreePlugin interface { type InTreePlugin interface {
// TranslateInTreeStorageClassParametersToCSI takes in-tree storage class
// parameters and translates them to a set of parameters consumable by CSI plugin
TranslateInTreeStorageClassParametersToCSI(scParameters map[string]string) (map[string]string, error)
// TranslateInTreePVToCSI takes a persistent volume and will translate // TranslateInTreePVToCSI takes a persistent volume and will translate
// the in-tree source to a CSI Source. The input persistent volume can be modified // the in-tree source to a CSI Source. The input persistent volume can be modified
TranslateInTreePVToCSI(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) TranslateInTreePVToCSI(pv *v1.PersistentVolume) (*v1.PersistentVolume, error)
......
...@@ -38,6 +38,11 @@ func NewOpenStackCinderCSITranslator() InTreePlugin { ...@@ -38,6 +38,11 @@ func NewOpenStackCinderCSITranslator() InTreePlugin {
return &osCinderCSITranslator{} return &osCinderCSITranslator{}
} }
// TranslateInTreeStorageClassParametersToCSI translates InTree Cinder storage class parameters to CSI storage class
func (t *osCinderCSITranslator) TranslateInTreeStorageClassParametersToCSI(scParameters map[string]string) (map[string]string, error) {
return scParameters, nil
}
// TranslateInTreePVToCSI takes a PV with Cinder set from in-tree // TranslateInTreePVToCSI takes a PV with Cinder set from in-tree
// and converts the Cinder source to a CSIPersistentVolumeSource // and converts the Cinder source to a CSIPersistentVolumeSource
func (t *osCinderCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) { func (t *osCinderCSITranslator) TranslateInTreePVToCSI(pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
......
...@@ -31,6 +31,17 @@ var ( ...@@ -31,6 +31,17 @@ var (
} }
) )
// TranslateInTreeStorageClassParametersToCSI takes in-tree storage class
// parameters and translates them to a set of parameters consumable by CSI plugin
func TranslateInTreeStorageClassParametersToCSI(inTreePluginName string, scParameters map[string]string) (map[string]string, error) {
for _, curPlugin := range inTreePlugins {
if inTreePluginName == curPlugin.GetInTreePluginName() {
return curPlugin.TranslateInTreeStorageClassParametersToCSI(scParameters)
}
}
return nil, fmt.Errorf("could not find in-tree storage class parameter translation logic for %#v", inTreePluginName)
}
// TranslateInTreePVToCSI takes a persistent volume and will translate // TranslateInTreePVToCSI takes a persistent volume and will translate
// the in-tree source to a CSI Source if the translation logic // the in-tree source to a CSI Source if the translation logic
// has been implemented. The input persistent volume will not // has been implemented. The input persistent volume will not
......
#!/usr/bin/env bash
# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
# Runs tests related to kubectl delete --all-namespaces.
run_kubectl_delete_allnamespaces_tests() {
set -o nounset
set -o errexit
ns_one="namespace-$(date +%s)-${RANDOM}"
ns_two="namespace-$(date +%s)-${RANDOM}"
kubectl create namespace "${ns_one}"
kubectl create namespace "${ns_two}"
kubectl create configmap "one" --namespace="${ns_one}"
kubectl create configmap "two" --namespace="${ns_two}"
kubectl label configmap "one" --namespace="${ns_one}" deletetest=true
kubectl label configmap "two" --namespace="${ns_two}" deletetest=true
kubectl delete configmap -l deletetest=true --all-namespaces
# no configmaps should be in either of those namespaces
kubectl config set-context "${CONTEXT}" --namespace="${ns_one}"
kube::test::get_object_assert configmap "{{range.items}}{{$id_field}}:{{end}}" ''
kubectl config set-context "${CONTEXT}" --namespace="${ns_two}"
kube::test::get_object_assert configmap "{{range.items}}{{$id_field}}:{{end}}" ''
set +o nounset
set +o errexit
}
...@@ -35,6 +35,7 @@ source "${KUBE_ROOT}/test/cmd/certificate.sh" ...@@ -35,6 +35,7 @@ source "${KUBE_ROOT}/test/cmd/certificate.sh"
source "${KUBE_ROOT}/test/cmd/core.sh" source "${KUBE_ROOT}/test/cmd/core.sh"
source "${KUBE_ROOT}/test/cmd/crd.sh" source "${KUBE_ROOT}/test/cmd/crd.sh"
source "${KUBE_ROOT}/test/cmd/create.sh" source "${KUBE_ROOT}/test/cmd/create.sh"
source "${KUBE_ROOT}/test/cmd/delete.sh"
source "${KUBE_ROOT}/test/cmd/diff.sh" source "${KUBE_ROOT}/test/cmd/diff.sh"
source "${KUBE_ROOT}/test/cmd/discovery.sh" source "${KUBE_ROOT}/test/cmd/discovery.sh"
source "${KUBE_ROOT}/test/cmd/generic-resources.sh" source "${KUBE_ROOT}/test/cmd/generic-resources.sh"
...@@ -499,6 +500,13 @@ runTests() { ...@@ -499,6 +500,13 @@ runTests() {
record_command run_create_secret_tests record_command run_create_secret_tests
fi fi
######################
# Delete #
######################
if kube::test::if_supports_resource "${configmaps}" ; then
record_command run_kubectl_delete_allnamespaces_tests
fi
################## ##################
# Global timeout # # Global timeout #
################## ##################
......
...@@ -69,6 +69,7 @@ filegroup( ...@@ -69,6 +69,7 @@ filegroup(
"//vendor/github.com/bazelbuild/buildtools/tables:all-srcs", "//vendor/github.com/bazelbuild/buildtools/tables:all-srcs",
"//vendor/github.com/beorn7/perks/quantile:all-srcs", "//vendor/github.com/beorn7/perks/quantile:all-srcs",
"//vendor/github.com/blang/semver:all-srcs", "//vendor/github.com/blang/semver:all-srcs",
"//vendor/github.com/cespare/prettybench:all-srcs",
"//vendor/github.com/chai2010/gettext-go/gettext:all-srcs", "//vendor/github.com/chai2010/gettext-go/gettext:all-srcs",
"//vendor/github.com/client9/misspell:all-srcs", "//vendor/github.com/client9/misspell:all-srcs",
"//vendor/github.com/cloudflare/cfssl/api:all-srcs", "//vendor/github.com/cloudflare/cfssl/api:all-srcs",
...@@ -285,6 +286,7 @@ filegroup( ...@@ -285,6 +286,7 @@ filegroup(
"//vendor/github.com/jmoiron/sqlx:all-srcs", "//vendor/github.com/jmoiron/sqlx:all-srcs",
"//vendor/github.com/jonboulle/clockwork:all-srcs", "//vendor/github.com/jonboulle/clockwork:all-srcs",
"//vendor/github.com/json-iterator/go:all-srcs", "//vendor/github.com/json-iterator/go:all-srcs",
"//vendor/github.com/jstemmer/go-junit-report:all-srcs",
"//vendor/github.com/jteeuwen/go-bindata:all-srcs", "//vendor/github.com/jteeuwen/go-bindata:all-srcs",
"//vendor/github.com/kardianos/osext:all-srcs", "//vendor/github.com/kardianos/osext:all-srcs",
"//vendor/github.com/karrick/godirwalk:all-srcs", "//vendor/github.com/karrick/godirwalk:all-srcs",
...@@ -419,9 +421,16 @@ filegroup( ...@@ -419,9 +421,16 @@ filegroup(
"//vendor/golang.org/x/tools/container/intsets:all-srcs", "//vendor/golang.org/x/tools/container/intsets:all-srcs",
"//vendor/golang.org/x/tools/go/ast/astutil:all-srcs", "//vendor/golang.org/x/tools/go/ast/astutil:all-srcs",
"//vendor/golang.org/x/tools/go/gcexportdata:all-srcs", "//vendor/golang.org/x/tools/go/gcexportdata:all-srcs",
"//vendor/golang.org/x/tools/go/gcimporter15:all-srcs", "//vendor/golang.org/x/tools/go/internal/cgo:all-srcs",
"//vendor/golang.org/x/tools/go/internal/gcimporter:all-srcs",
"//vendor/golang.org/x/tools/go/internal/packagesdriver:all-srcs",
"//vendor/golang.org/x/tools/go/packages:all-srcs",
"//vendor/golang.org/x/tools/go/vcs:all-srcs", "//vendor/golang.org/x/tools/go/vcs:all-srcs",
"//vendor/golang.org/x/tools/imports:all-srcs", "//vendor/golang.org/x/tools/imports:all-srcs",
"//vendor/golang.org/x/tools/internal/fastwalk:all-srcs",
"//vendor/golang.org/x/tools/internal/gopathwalk:all-srcs",
"//vendor/golang.org/x/tools/internal/module:all-srcs",
"//vendor/golang.org/x/tools/internal/semver:all-srcs",
"//vendor/gonum.org/v1/gonum/blas:all-srcs", "//vendor/gonum.org/v1/gonum/blas:all-srcs",
"//vendor/gonum.org/v1/gonum/floats:all-srcs", "//vendor/gonum.org/v1/gonum/floats:all-srcs",
"//vendor/gonum.org/v1/gonum/graph:all-srcs", "//vendor/gonum.org/v1/gonum/graph:all-srcs",
......
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
go_library(
name = "go_default_library",
srcs = ["prettybench.go"],
importmap = "k8s.io/kubernetes/vendor/github.com/cespare/prettybench",
importpath = "github.com/cespare/prettybench",
visibility = ["//visibility:private"],
deps = ["//vendor/golang.org/x/tools/benchmark/parse:go_default_library"],
)
go_binary(
name = "prettybench",
embed = [":go_default_library"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
Copyright (c) 2014 Caleb Spare
MIT License
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# Prettybench
A tool for transforming `go test`'s benchmark output a bit to make it nicer for humans.
## Problem
Go benchmarks are great, particularly when used in concert with benchcmp. But the output can be a bit hard to
read:
![before](/screenshots/before.png)
## Solution
$ go get github.com/cespare/prettybench
$ go test -bench=. | prettybench
![after](/screenshots/after.png)
* Column headers
* Columns are aligned
* Time output is adjusted to convenient units
## Notes
* Right now the units for the time are chosen based on the smallest value in the column.
* Prettybench has to buffer all the rows of output before it can print them (for column formatting), so you
won't see intermediate progress. If you want to see that too, you could tee your output so that you see the
unmodified version as well. If you do this, you'll want to use the prettybench's `-no-passthrough` flag so
it doesn't print all the other lines (because then they'd be printed twice):
$ go test -bench=. | tee >(prettybench -no-passthrough)
## To Do (maybe)
* Handle benchcmp output
* Change the units for non-time columns as well (these are generally OK though).
package main
import (
"bufio"
"bytes"
"errors"
"flag"
"fmt"
"os"
"regexp"
"strconv"
"strings"
"time"
bench "golang.org/x/tools/benchmark/parse"
)
var noPassthrough = flag.Bool("no-passthrough", false, "Don't print non-benchmark lines")
type BenchOutputGroup struct {
Lines []*bench.Benchmark
// Columns which are in use
Measured int
}
type Table struct {
MaxLengths []int
Cells [][]string
}
func (g *BenchOutputGroup) String() string {
if len(g.Lines) == 0 {
return ""
}
columnNames := []string{"benchmark", "iter", "time/iter"}
if (g.Measured & bench.MBPerS) > 0 {
columnNames = append(columnNames, "throughput")
}
if (g.Measured & bench.AllocedBytesPerOp) > 0 {
columnNames = append(columnNames, "bytes alloc")
}
if (g.Measured & bench.AllocsPerOp) > 0 {
columnNames = append(columnNames, "allocs")
}
table := &Table{Cells: [][]string{columnNames}}
var underlines []string
for _, name := range columnNames {
underlines = append(underlines, strings.Repeat("-", len(name)))
}
table.Cells = append(table.Cells, underlines)
timeFormatFunc := g.TimeFormatFunc()
for _, line := range g.Lines {
row := []string{line.Name, FormatIterations(line.N), timeFormatFunc(line.NsPerOp)}
if (g.Measured & bench.MBPerS) > 0 {
row = append(row, FormatMegaBytesPerSecond(line))
}
if (g.Measured & bench.AllocedBytesPerOp) > 0 {
row = append(row, FormatBytesAllocPerOp(line))
}
if (g.Measured & bench.AllocsPerOp) > 0 {
row = append(row, FormatAllocsPerOp(line))
}
table.Cells = append(table.Cells, row)
}
for i := range columnNames {
maxLength := 0
for _, row := range table.Cells {
if len(row[i]) > maxLength {
maxLength = len(row[i])
}
}
table.MaxLengths = append(table.MaxLengths, maxLength)
}
var buf bytes.Buffer
for _, row := range table.Cells {
for i, cell := range row {
var format string
switch i {
case 0:
format = "%%-%ds "
case len(row) - 1:
format = "%%%ds"
default:
format = "%%%ds "
}
fmt.Fprintf(&buf, fmt.Sprintf(format, table.MaxLengths[i]), cell)
}
fmt.Fprint(&buf, "\n")
}
return buf.String()
}
func FormatIterations(iter int) string {
return strconv.FormatInt(int64(iter), 10)
}
func (g *BenchOutputGroup) TimeFormatFunc() func(float64) string {
// Find the smallest time
smallest := g.Lines[0].NsPerOp
for _, line := range g.Lines[1:] {
if line.NsPerOp < smallest {
smallest = line.NsPerOp
}
}
switch {
case smallest < float64(10000*time.Nanosecond):
return func(ns float64) string {
return fmt.Sprintf("%.2f ns/op", ns)
}
case smallest < float64(time.Millisecond):
return func(ns float64) string {
return fmt.Sprintf("%.2f μs/op", ns/1000)
}
case smallest < float64(10*time.Second):
return func(ns float64) string {
return fmt.Sprintf("%.2f ms/op", (ns / 1e6))
}
default:
return func(ns float64) string {
return fmt.Sprintf("%.2f s/op", ns/1e9)
}
}
}
func FormatMegaBytesPerSecond(l *bench.Benchmark) string {
if (l.Measured & bench.MBPerS) == 0 {
return ""
}
return fmt.Sprintf("%.2f MB/s", l.MBPerS)
}
func FormatBytesAllocPerOp(l *bench.Benchmark) string {
if (l.Measured & bench.AllocedBytesPerOp) == 0 {
return ""
}
return fmt.Sprintf("%d B/op", l.AllocedBytesPerOp)
}
func FormatAllocsPerOp(l *bench.Benchmark) string {
if (l.Measured & bench.AllocsPerOp) == 0 {
return ""
}
return fmt.Sprintf("%d allocs/op", l.AllocsPerOp)
}
func (g *BenchOutputGroup) AddLine(line *bench.Benchmark) {
g.Lines = append(g.Lines, line)
g.Measured |= line.Measured
}
var (
benchLineMatcher = regexp.MustCompile(`^Benchmark.*\t.*\d+`)
okLineMatcher = regexp.MustCompile(`^ok\s`)
notBenchLineErr = errors.New("Not a bench line")
)
func ParseLine(line string) (*bench.Benchmark, error) {
if !benchLineMatcher.MatchString(line) {
return nil, notBenchLineErr
}
fields := strings.Split(line, "\t")
if len(fields) < 3 {
return nil, notBenchLineErr
}
return bench.ParseLine(line)
}
func main() {
flag.Parse()
currentBenchmark := &BenchOutputGroup{}
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
text := scanner.Text()
line, err := ParseLine(text)
switch err {
case notBenchLineErr:
if okLineMatcher.MatchString(text) {
fmt.Print(currentBenchmark)
currentBenchmark = &BenchOutputGroup{}
}
if !*noPassthrough {
fmt.Println(text)
}
case nil:
currentBenchmark.AddLine(line)
default:
fmt.Fprintln(os.Stderr, "prettybench unrecognized line:")
fmt.Println(text)
}
}
if err := scanner.Err(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
language: go
go:
- tip
- "1.10.x"
- "1.9.x"
- "1.8.x"
- "1.7.x"
- "1.6.x"
- "1.5.x"
- "1.4.x"
- "1.3.x"
- "1.2.x"
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
go_library(
name = "go_default_library",
srcs = ["go-junit-report.go"],
importmap = "k8s.io/kubernetes/vendor/github.com/jstemmer/go-junit-report",
importpath = "github.com/jstemmer/go-junit-report",
visibility = ["//visibility:private"],
deps = [
"//vendor/github.com/jstemmer/go-junit-report/formatter:go_default_library",
"//vendor/github.com/jstemmer/go-junit-report/parser:go_default_library",
],
)
go_binary(
name = "go-junit-report",
embed = [":go_default_library"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//vendor/github.com/jstemmer/go-junit-report/formatter:all-srcs",
"//vendor/github.com/jstemmer/go-junit-report/parser:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
Copyright (c) 2012 Joel Stemmer
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# go-junit-report
Converts `go test` output to an xml report, suitable for applications that
expect junit xml reports (e.g. [Jenkins](http://jenkins-ci.org)).
[![Build Status][travis-badge]][travis-link]
[![Report Card][report-badge]][report-link]
## Installation
Go version 1.1 or higher is required. Install or update using the `go get`
command:
```bash
go get -u github.com/jstemmer/go-junit-report
```
## Contribution
Create an Issue and discuss the fix or feature, then fork the package.
Clone to github.com/jstemmer/go-junit-report. This is necessary because go import uses this path.
Fix or implement feature. Test and then commit change.
Specify #Issue and describe change in the commit message.
Create Pull Request. It can be merged by owner or administrator then.
## Run Tests
go test
## Usage
go-junit-report reads the `go test` verbose output from standard in and writes
junit compatible XML to standard out.
```bash
go test -v 2>&1 | go-junit-report > report.xml
```
Note that it also can parse benchmark output with `-bench` flag:
```bash
go test -v -bench . -count 5 2>&1 | go-junit-report > report.xml
```
[travis-badge]: https://travis-ci.org/jstemmer/go-junit-report.svg
[travis-link]: https://travis-ci.org/jstemmer/go-junit-report
[report-badge]: https://goreportcard.com/badge/github.com/jstemmer/go-junit-report
[report-link]: https://goreportcard.com/report/github.com/jstemmer/go-junit-report
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["formatter.go"],
importmap = "k8s.io/kubernetes/vendor/github.com/jstemmer/go-junit-report/formatter",
importpath = "github.com/jstemmer/go-junit-report/formatter",
visibility = ["//visibility:public"],
deps = ["//vendor/github.com/jstemmer/go-junit-report/parser:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
package formatter
import (
"bufio"
"encoding/xml"
"fmt"
"io"
"runtime"
"strings"
"time"
"github.com/jstemmer/go-junit-report/parser"
)
// JUnitTestSuites is a collection of JUnit test suites.
type JUnitTestSuites struct {
XMLName xml.Name `xml:"testsuites"`
Suites []JUnitTestSuite
}
// JUnitTestSuite is a single JUnit test suite which may contain many
// testcases.
type JUnitTestSuite struct {
XMLName xml.Name `xml:"testsuite"`
Tests int `xml:"tests,attr"`
Failures int `xml:"failures,attr"`
Time string `xml:"time,attr"`
Name string `xml:"name,attr"`
Properties []JUnitProperty `xml:"properties>property,omitempty"`
TestCases []JUnitTestCase
}
// JUnitTestCase is a single test case with its result.
type JUnitTestCase struct {
XMLName xml.Name `xml:"testcase"`
Classname string `xml:"classname,attr"`
Name string `xml:"name,attr"`
Time string `xml:"time,attr"`
SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"`
Failure *JUnitFailure `xml:"failure,omitempty"`
}
// JUnitSkipMessage contains the reason why a testcase was skipped.
type JUnitSkipMessage struct {
Message string `xml:"message,attr"`
}
// JUnitProperty represents a key/value pair used to define properties.
type JUnitProperty struct {
Name string `xml:"name,attr"`
Value string `xml:"value,attr"`
}
// JUnitFailure contains data related to a failed test.
type JUnitFailure struct {
Message string `xml:"message,attr"`
Type string `xml:"type,attr"`
Contents string `xml:",chardata"`
}
// JUnitReportXML writes a JUnit xml representation of the given report to w
// in the format described at http://windyroad.org/dl/Open%20Source/JUnit.xsd
func JUnitReportXML(report *parser.Report, noXMLHeader bool, goVersion string, w io.Writer) error {
suites := JUnitTestSuites{}
// convert Report to JUnit test suites
for _, pkg := range report.Packages {
pkg.Benchmarks = mergeBenchmarks(pkg.Benchmarks)
ts := JUnitTestSuite{
Tests: len(pkg.Tests) + len(pkg.Benchmarks),
Failures: 0,
Time: formatTime(pkg.Duration),
Name: pkg.Name,
Properties: []JUnitProperty{},
TestCases: []JUnitTestCase{},
}
classname := pkg.Name
if idx := strings.LastIndex(classname, "/"); idx > -1 && idx < len(pkg.Name) {
classname = pkg.Name[idx+1:]
}
// properties
if goVersion == "" {
// if goVersion was not specified as a flag, fall back to version reported by runtime
goVersion = runtime.Version()
}
ts.Properties = append(ts.Properties, JUnitProperty{"go.version", goVersion})
if pkg.CoveragePct != "" {
ts.Properties = append(ts.Properties, JUnitProperty{"coverage.statements.pct", pkg.CoveragePct})
}
// individual test cases
for _, test := range pkg.Tests {
testCase := JUnitTestCase{
Classname: classname,
Name: test.Name,
Time: formatTime(test.Duration),
Failure: nil,
}
if test.Result == parser.FAIL {
ts.Failures++
testCase.Failure = &JUnitFailure{
Message: "Failed",
Type: "",
Contents: strings.Join(test.Output, "\n"),
}
}
if test.Result == parser.SKIP {
testCase.SkipMessage = &JUnitSkipMessage{strings.Join(test.Output, "\n")}
}
ts.TestCases = append(ts.TestCases, testCase)
}
// individual benchmarks
for _, benchmark := range pkg.Benchmarks {
benchmarkCase := JUnitTestCase{
Classname: classname,
Name: benchmark.Name,
Time: formatBenchmarkTime(benchmark.Duration),
}
ts.TestCases = append(ts.TestCases, benchmarkCase)
}
suites.Suites = append(suites.Suites, ts)
}
// to xml
bytes, err := xml.MarshalIndent(suites, "", "\t")
if err != nil {
return err
}
writer := bufio.NewWriter(w)
if !noXMLHeader {
writer.WriteString(xml.Header)
}
writer.Write(bytes)
writer.WriteByte('\n')
writer.Flush()
return nil
}
func mergeBenchmarks(benchmarks []*parser.Benchmark) []*parser.Benchmark {
var merged []*parser.Benchmark
benchmap := make(map[string][]*parser.Benchmark)
for _, bm := range benchmarks {
if _, ok := benchmap[bm.Name]; !ok {
merged = append(merged, &parser.Benchmark{Name: bm.Name})
}
benchmap[bm.Name] = append(benchmap[bm.Name], bm)
}
for _, bm := range merged {
for _, b := range benchmap[bm.Name] {
bm.Allocs += b.Allocs
bm.Bytes += b.Bytes
bm.Duration += b.Duration
}
n := len(benchmap[bm.Name])
bm.Allocs /= n
bm.Bytes /= n
bm.Duration /= time.Duration(n)
}
return merged
}
func formatTime(d time.Duration) string {
return fmt.Sprintf("%.3f", d.Seconds())
}
func formatBenchmarkTime(d time.Duration) string {
return fmt.Sprintf("%.9f", d.Seconds())
}
package main
import (
"flag"
"fmt"
"os"
"github.com/jstemmer/go-junit-report/formatter"
"github.com/jstemmer/go-junit-report/parser"
)
var (
noXMLHeader bool
packageName string
goVersionFlag string
setExitCode bool
)
func init() {
flag.BoolVar(&noXMLHeader, "no-xml-header", false, "do not print xml header")
flag.StringVar(&packageName, "package-name", "", "specify a package name (compiled test have no package name in output)")
flag.StringVar(&goVersionFlag, "go-version", "", "specify the value to use for the go.version property in the generated XML")
flag.BoolVar(&setExitCode, "set-exit-code", false, "set exit code to 1 if tests failed")
}
func main() {
flag.Parse()
if flag.NArg() != 0 {
fmt.Println("go-junit-report does not accept positional arguments")
os.Exit(1)
}
// Read input
report, err := parser.Parse(os.Stdin, packageName)
if err != nil {
fmt.Printf("Error reading input: %s\n", err)
os.Exit(1)
}
// Write xml
err = formatter.JUnitReportXML(report, noXMLHeader, goVersionFlag, os.Stdout)
if err != nil {
fmt.Printf("Error writing XML: %s\n", err)
os.Exit(1)
}
if setExitCode && report.Failures() > 0 {
os.Exit(1)
}
}
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["parser.go"],
importmap = "k8s.io/kubernetes/vendor/github.com/jstemmer/go-junit-report/parser",
importpath = "github.com/jstemmer/go-junit-report/parser",
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
package parser
import (
"bufio"
"io"
"regexp"
"strconv"
"strings"
"time"
)
// Result represents a test result.
type Result int
// Test result constants
const (
PASS Result = iota
FAIL
SKIP
)
// Report is a collection of package tests.
type Report struct {
Packages []Package
}
// Package contains the test results of a single package.
type Package struct {
Name string
Duration time.Duration
Tests []*Test
Benchmarks []*Benchmark
CoveragePct string
// Time is deprecated, use Duration instead.
Time int // in milliseconds
}
// Test contains the results of a single test.
type Test struct {
Name string
Duration time.Duration
Result Result
Output []string
SubtestIndent string
// Time is deprecated, use Duration instead.
Time int // in milliseconds
}
// Benchmark contains the results of a single benchmark.
type Benchmark struct {
Name string
Duration time.Duration
// number of B/op
Bytes int
// number of allocs/op
Allocs int
}
var (
regexStatus = regexp.MustCompile(`--- (PASS|FAIL|SKIP): (.+) \((\d+\.\d+)(?: seconds|s)\)`)
regexIndent = regexp.MustCompile(`^([ \t]+)---`)
regexCoverage = regexp.MustCompile(`^coverage:\s+(\d+\.\d+)%\s+of\s+statements(?:\sin\s.+)?$`)
regexResult = regexp.MustCompile(`^(ok|FAIL)\s+([^ ]+)\s+(?:(\d+\.\d+)s|\(cached\)|(\[\w+ failed]))(?:\s+coverage:\s+(\d+\.\d+)%\sof\sstatements(?:\sin\s.+)?)?$`)
// regexBenchmark captures 3-5 groups: benchmark name, number of times ran, ns/op (with or without decimal), B/op (optional), and allocs/op (optional).
regexBenchmark = regexp.MustCompile(`^(Benchmark[^ -]+)(?:-\d+\s+|\s+)(\d+)\s+(\d+|\d+\.\d+)\sns/op(?:\s+(\d+)\sB/op)?(?:\s+(\d+)\sallocs/op)?`)
regexOutput = regexp.MustCompile(`( )*\t(.*)`)
regexSummary = regexp.MustCompile(`^(PASS|FAIL|SKIP)$`)
)
// Parse parses go test output from reader r and returns a report with the
// results. An optional pkgName can be given, which is used in case a package
// result line is missing.
func Parse(r io.Reader, pkgName string) (*Report, error) {
reader := bufio.NewReader(r)
report := &Report{make([]Package, 0)}
// keep track of tests we find
var tests []*Test
// keep track of benchmarks we find
var benchmarks []*Benchmark
// sum of tests' time, use this if current test has no result line (when it is compiled test)
var testsTime time.Duration
// current test
var cur string
// keep track if we've already seen a summary for the current test
var seenSummary bool
// coverage percentage report for current package
var coveragePct string
// stores mapping between package name and output of build failures
var packageCaptures = map[string][]string{}
// the name of the package which it's build failure output is being captured
var capturedPackage string
// capture any non-test output
var buffers = map[string][]string{}
// parse lines
for {
l, _, err := reader.ReadLine()
if err != nil && err == io.EOF {
break
} else if err != nil {
return nil, err
}
line := string(l)
if strings.HasPrefix(line, "=== RUN ") {
// new test
cur = strings.TrimSpace(line[8:])
tests = append(tests, &Test{
Name: cur,
Result: FAIL,
Output: make([]string, 0),
})
// clear the current build package, so output lines won't be added to that build
capturedPackage = ""
seenSummary = false
} else if matches := regexBenchmark.FindStringSubmatch(line); len(matches) == 6 {
bytes, _ := strconv.Atoi(matches[4])
allocs, _ := strconv.Atoi(matches[5])
benchmarks = append(benchmarks, &Benchmark{
Name: matches[1],
Duration: parseNanoseconds(matches[3]),
Bytes: bytes,
Allocs: allocs,
})
} else if strings.HasPrefix(line, "=== PAUSE ") {
continue
} else if strings.HasPrefix(line, "=== CONT ") {
cur = strings.TrimSpace(line[8:])
continue
} else if matches := regexResult.FindStringSubmatch(line); len(matches) == 6 {
if matches[5] != "" {
coveragePct = matches[5]
}
if strings.HasSuffix(matches[4], "failed]") {
// the build of the package failed, inject a dummy test into the package
// which indicate about the failure and contain the failure description.
tests = append(tests, &Test{
Name: matches[4],
Result: FAIL,
Output: packageCaptures[matches[2]],
})
} else if matches[1] == "FAIL" && len(tests) == 0 && len(buffers[cur]) > 0 {
// This package didn't have any tests, but it failed with some
// output. Create a dummy test with the output.
tests = append(tests, &Test{
Name: "Failure",
Result: FAIL,
Output: buffers[cur],
})
buffers[cur] = buffers[cur][0:0]
}
// all tests in this package are finished
report.Packages = append(report.Packages, Package{
Name: matches[2],
Duration: parseSeconds(matches[3]),
Tests: tests,
Benchmarks: benchmarks,
CoveragePct: coveragePct,
Time: int(parseSeconds(matches[3]) / time.Millisecond), // deprecated
})
buffers[cur] = buffers[cur][0:0]
tests = make([]*Test, 0)
benchmarks = make([]*Benchmark, 0)
coveragePct = ""
cur = ""
testsTime = 0
} else if matches := regexStatus.FindStringSubmatch(line); len(matches) == 4 {
cur = matches[2]
test := findTest(tests, cur)
if test == nil {
continue
}
// test status
if matches[1] == "PASS" {
test.Result = PASS
} else if matches[1] == "SKIP" {
test.Result = SKIP
} else {
test.Result = FAIL
}
if matches := regexIndent.FindStringSubmatch(line); len(matches) == 2 {
test.SubtestIndent = matches[1]
}
test.Output = buffers[cur]
test.Name = matches[2]
test.Duration = parseSeconds(matches[3])
testsTime += test.Duration
test.Time = int(test.Duration / time.Millisecond) // deprecated
} else if matches := regexCoverage.FindStringSubmatch(line); len(matches) == 2 {
coveragePct = matches[1]
} else if matches := regexOutput.FindStringSubmatch(line); capturedPackage == "" && len(matches) == 3 {
// Sub-tests start with one or more series of 4-space indents, followed by a hard tab,
// followed by the test output
// Top-level tests start with a hard tab.
test := findTest(tests, cur)
if test == nil {
continue
}
test.Output = append(test.Output, matches[2])
} else if strings.HasPrefix(line, "# ") {
// indicates a capture of build output of a package. set the current build package.
capturedPackage = line[2:]
} else if capturedPackage != "" {
// current line is build failure capture for the current built package
packageCaptures[capturedPackage] = append(packageCaptures[capturedPackage], line)
} else if regexSummary.MatchString(line) {
// don't store any output after the summary
seenSummary = true
} else if !seenSummary {
// buffer anything else that we didn't recognize
buffers[cur] = append(buffers[cur], line)
// if we have a current test, also append to its output
test := findTest(tests, cur)
if test != nil {
if strings.HasPrefix(line, test.SubtestIndent+" ") {
test.Output = append(test.Output, strings.TrimPrefix(line, test.SubtestIndent+" "))
}
}
}
}
if len(tests) > 0 {
// no result line found
report.Packages = append(report.Packages, Package{
Name: pkgName,
Duration: testsTime,
Time: int(testsTime / time.Millisecond),
Tests: tests,
Benchmarks: benchmarks,
CoveragePct: coveragePct,
})
}
return report, nil
}
func parseSeconds(t string) time.Duration {
if t == "" {
return time.Duration(0)
}
// ignore error
d, _ := time.ParseDuration(t + "s")
return d
}
func parseNanoseconds(t string) time.Duration {
// note: if input < 1 ns precision, result will be 0s.
if t == "" {
return time.Duration(0)
}
// ignore error
d, _ := time.ParseDuration(t + "ns")
return d
}
func findTest(tests []*Test, name string) *Test {
for i := len(tests) - 1; i >= 0; i-- {
if tests[i].Name == name {
return tests[i]
}
}
return nil
}
// Failures counts the number of failed tests in this report
func (r *Report) Failures() int {
count := 0
for _, p := range r.Packages {
for _, t := range p.Tests {
if t.Result == FAIL {
count++
}
}
}
return count
}
...@@ -5,6 +5,7 @@ go_library( ...@@ -5,6 +5,7 @@ go_library(
srcs = [ srcs = [
"enclosing.go", "enclosing.go",
"imports.go", "imports.go",
"rewrite.go",
"util.go", "util.go",
], ],
importmap = "k8s.io/kubernetes/vendor/golang.org/x/tools/go/ast/astutil", importmap = "k8s.io/kubernetes/vendor/golang.org/x/tools/go/ast/astutil",
......
...@@ -14,26 +14,26 @@ import ( ...@@ -14,26 +14,26 @@ import (
) )
// AddImport adds the import path to the file f, if absent. // AddImport adds the import path to the file f, if absent.
func AddImport(fset *token.FileSet, f *ast.File, ipath string) (added bool) { func AddImport(fset *token.FileSet, f *ast.File, path string) (added bool) {
return AddNamedImport(fset, f, "", ipath) return AddNamedImport(fset, f, "", path)
} }
// AddNamedImport adds the import path to the file f, if absent. // AddNamedImport adds the import with the given name and path to the file f, if absent.
// If name is not empty, it is used to rename the import. // If name is not empty, it is used to rename the import.
// //
// For example, calling // For example, calling
// AddNamedImport(fset, f, "pathpkg", "path") // AddNamedImport(fset, f, "pathpkg", "path")
// adds // adds
// import pathpkg "path" // import pathpkg "path"
func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added bool) { func AddNamedImport(fset *token.FileSet, f *ast.File, name, path string) (added bool) {
if imports(f, ipath) { if imports(f, name, path) {
return false return false
} }
newImport := &ast.ImportSpec{ newImport := &ast.ImportSpec{
Path: &ast.BasicLit{ Path: &ast.BasicLit{
Kind: token.STRING, Kind: token.STRING,
Value: strconv.Quote(ipath), Value: strconv.Quote(path),
}, },
} }
if name != "" { if name != "" {
...@@ -43,12 +43,14 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added ...@@ -43,12 +43,14 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added
// Find an import decl to add to. // Find an import decl to add to.
// The goal is to find an existing import // The goal is to find an existing import
// whose import path has the longest shared // whose import path has the longest shared
// prefix with ipath. // prefix with path.
var ( var (
bestMatch = -1 // length of longest shared prefix bestMatch = -1 // length of longest shared prefix
lastImport = -1 // index in f.Decls of the file's final import decl lastImport = -1 // index in f.Decls of the file's final import decl
impDecl *ast.GenDecl // import decl containing the best match impDecl *ast.GenDecl // import decl containing the best match
impIndex = -1 // spec index in impDecl containing the best match impIndex = -1 // spec index in impDecl containing the best match
isThirdPartyPath = isThirdParty(path)
) )
for i, decl := range f.Decls { for i, decl := range f.Decls {
gen, ok := decl.(*ast.GenDecl) gen, ok := decl.(*ast.GenDecl)
...@@ -65,15 +67,27 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added ...@@ -65,15 +67,27 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added
impDecl = gen impDecl = gen
} }
// Compute longest shared prefix with imports in this group. // Compute longest shared prefix with imports in this group and find best
// matched import spec.
// 1. Always prefer import spec with longest shared prefix.
// 2. While match length is 0,
// - for stdlib package: prefer first import spec.
// - for third party package: prefer first third party import spec.
// We cannot use last import spec as best match for third party package
// because grouped imports are usually placed last by goimports -local
// flag.
// See issue #19190.
seenAnyThirdParty := false
for j, spec := range gen.Specs { for j, spec := range gen.Specs {
impspec := spec.(*ast.ImportSpec) impspec := spec.(*ast.ImportSpec)
n := matchLen(importPath(impspec), ipath) p := importPath(impspec)
if n > bestMatch { n := matchLen(p, path)
if n > bestMatch || (bestMatch == 0 && !seenAnyThirdParty && isThirdPartyPath) {
bestMatch = n bestMatch = n
impDecl = gen impDecl = gen
impIndex = j impIndex = j
} }
seenAnyThirdParty = seenAnyThirdParty || isThirdParty(p)
} }
} }
} }
...@@ -87,8 +101,8 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added ...@@ -87,8 +101,8 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added
impDecl.TokPos = f.Decls[lastImport].End() impDecl.TokPos = f.Decls[lastImport].End()
} else { } else {
// There are no existing imports. // There are no existing imports.
// Our new import goes after the package declaration and after // Our new import, preceded by a blank line, goes after the package declaration
// the comment, if any, that starts on the same line as the // and after the comment, if any, that starts on the same line as the
// package declaration. // package declaration.
impDecl.TokPos = f.Package impDecl.TokPos = f.Package
...@@ -98,7 +112,8 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added ...@@ -98,7 +112,8 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added
if file.Line(c.Pos()) > pkgLine { if file.Line(c.Pos()) > pkgLine {
break break
} }
impDecl.TokPos = c.End() // +2 for a blank line
impDecl.TokPos = c.End() + 2
} }
} }
f.Decls = append(f.Decls, nil) f.Decls = append(f.Decls, nil)
...@@ -175,12 +190,20 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added ...@@ -175,12 +190,20 @@ func AddNamedImport(fset *token.FileSet, f *ast.File, name, ipath string) (added
return true return true
} }
func isThirdParty(importPath string) bool {
// Third party package import path usually contains "." (".com", ".org", ...)
// This logic is taken from golang.org/x/tools/imports package.
return strings.Contains(importPath, ".")
}
// DeleteImport deletes the import path from the file f, if present. // DeleteImport deletes the import path from the file f, if present.
// If there are duplicate import declarations, all matching ones are deleted.
func DeleteImport(fset *token.FileSet, f *ast.File, path string) (deleted bool) { func DeleteImport(fset *token.FileSet, f *ast.File, path string) (deleted bool) {
return DeleteNamedImport(fset, f, "", path) return DeleteNamedImport(fset, f, "", path)
} }
// DeleteNamedImport deletes the import with the given name and path from the file f, if present. // DeleteNamedImport deletes the import with the given name and path from the file f, if present.
// If there are duplicate import declarations, all matching ones are deleted.
func DeleteNamedImport(fset *token.FileSet, f *ast.File, name, path string) (deleted bool) { func DeleteNamedImport(fset *token.FileSet, f *ast.File, name, path string) (deleted bool) {
var delspecs []*ast.ImportSpec var delspecs []*ast.ImportSpec
var delcomments []*ast.CommentGroup var delcomments []*ast.CommentGroup
...@@ -195,13 +218,7 @@ func DeleteNamedImport(fset *token.FileSet, f *ast.File, name, path string) (del ...@@ -195,13 +218,7 @@ func DeleteNamedImport(fset *token.FileSet, f *ast.File, name, path string) (del
for j := 0; j < len(gen.Specs); j++ { for j := 0; j < len(gen.Specs); j++ {
spec := gen.Specs[j] spec := gen.Specs[j]
impspec := spec.(*ast.ImportSpec) impspec := spec.(*ast.ImportSpec)
if impspec.Name == nil && name != "" { if importName(impspec) != name || importPath(impspec) != path {
continue
}
if impspec.Name != nil && impspec.Name.Name != name {
continue
}
if importPath(impspec) != path {
continue continue
} }
...@@ -262,7 +279,7 @@ func DeleteNamedImport(fset *token.FileSet, f *ast.File, name, path string) (del ...@@ -262,7 +279,7 @@ func DeleteNamedImport(fset *token.FileSet, f *ast.File, name, path string) (del
// There was a blank line immediately preceding the deleted import, // There was a blank line immediately preceding the deleted import,
// so there's no need to close the hole. // so there's no need to close the hole.
// Do nothing. // Do nothing.
} else { } else if line != fset.File(gen.Rparen).LineCount() {
// There was no blank line. Close the hole. // There was no blank line. Close the hole.
fset.File(gen.Rparen).MergeLine(line) fset.File(gen.Rparen).MergeLine(line)
} }
...@@ -362,9 +379,14 @@ func (fn visitFn) Visit(node ast.Node) ast.Visitor { ...@@ -362,9 +379,14 @@ func (fn visitFn) Visit(node ast.Node) ast.Visitor {
return fn return fn
} }
// imports returns true if f imports path. // imports reports whether f has an import with the specified name and path.
func imports(f *ast.File, path string) bool { func imports(f *ast.File, name, path string) bool {
return importSpec(f, path) != nil for _, s := range f.Imports {
if importName(s) == name && importPath(s) == path {
return true
}
}
return false
} }
// importSpec returns the import spec if f imports path, // importSpec returns the import spec if f imports path,
...@@ -378,14 +400,23 @@ func importSpec(f *ast.File, path string) *ast.ImportSpec { ...@@ -378,14 +400,23 @@ func importSpec(f *ast.File, path string) *ast.ImportSpec {
return nil return nil
} }
// importName returns the name of s,
// or "" if the import is not named.
func importName(s *ast.ImportSpec) string {
if s.Name == nil {
return ""
}
return s.Name.Name
}
// importPath returns the unquoted import path of s, // importPath returns the unquoted import path of s,
// or "" if the path is not properly quoted. // or "" if the path is not properly quoted.
func importPath(s *ast.ImportSpec) string { func importPath(s *ast.ImportSpec) string {
t, err := strconv.Unquote(s.Path.Value) t, err := strconv.Unquote(s.Path.Value)
if err == nil { if err != nil {
return t return ""
} }
return "" return t
} }
// declImports reports whether gen contains an import of path. // declImports reports whether gen contains an import of path.
......
...@@ -9,7 +9,7 @@ go_library( ...@@ -9,7 +9,7 @@ go_library(
importmap = "k8s.io/kubernetes/vendor/golang.org/x/tools/go/gcexportdata", importmap = "k8s.io/kubernetes/vendor/golang.org/x/tools/go/gcexportdata",
importpath = "golang.org/x/tools/go/gcexportdata", importpath = "golang.org/x/tools/go/gcexportdata",
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
deps = ["//vendor/golang.org/x/tools/go/gcimporter15:go_default_library"], deps = ["//vendor/golang.org/x/tools/go/internal/gcimporter:go_default_library"],
) )
filegroup( filegroup(
......
...@@ -7,9 +7,6 @@ ...@@ -7,9 +7,6 @@
// gc compiler. This package supports go1.7 export data format and all // gc compiler. This package supports go1.7 export data format and all
// later versions. // later versions.
// //
// This package replaces the deprecated golang.org/x/tools/go/gcimporter15
// package, which will be deleted in October 2017.
//
// Although it might seem convenient for this package to live alongside // Although it might seem convenient for this package to live alongside
// go/types in the standard library, this would cause version skew // go/types in the standard library, this would cause version skew
// problems for developer tools that use it, since they must be able to // problems for developer tools that use it, since they must be able to
...@@ -19,7 +16,7 @@ ...@@ -19,7 +16,7 @@
// time before the Go 1.8 release and rebuild and redeploy their // time before the Go 1.8 release and rebuild and redeploy their
// developer tools, which will then be able to consume both Go 1.7 and // developer tools, which will then be able to consume both Go 1.7 and
// Go 1.8 export data files, so they will work before and after the // Go 1.8 export data files, so they will work before and after the
// Go update. (See discussion at https://github.com/golang/go/issues/15651.) // Go update. (See discussion at https://golang.org/issue/15651.)
// //
package gcexportdata package gcexportdata
...@@ -32,7 +29,7 @@ import ( ...@@ -32,7 +29,7 @@ import (
"io" "io"
"io/ioutil" "io/ioutil"
gcimporter "golang.org/x/tools/go/gcimporter15" "golang.org/x/tools/go/internal/gcimporter"
) )
// Find returns the name of an object (.o) or archive (.a) file // Find returns the name of an object (.o) or archive (.a) file
...@@ -44,7 +41,7 @@ import ( ...@@ -44,7 +41,7 @@ import (
// //
// Find also returns the package's resolved (canonical) import path, // Find also returns the package's resolved (canonical) import path,
// reflecting the effects of srcDir and vendoring on importPath. // reflecting the effects of srcDir and vendoring on importPath.
func Find(importPath string, srcDir string) (filename, path string) { func Find(importPath, srcDir string) (filename, path string) {
return gcimporter.FindPkg(importPath, srcDir) return gcimporter.FindPkg(importPath, srcDir)
} }
...@@ -88,6 +85,14 @@ func Read(in io.Reader, fset *token.FileSet, imports map[string]*types.Package, ...@@ -88,6 +85,14 @@ func Read(in io.Reader, fset *token.FileSet, imports map[string]*types.Package,
return gcimporter.ImportData(imports, path, path, bytes.NewReader(data)) return gcimporter.ImportData(imports, path, path, bytes.NewReader(data))
} }
// The indexed export format starts with an 'i'; the older
// binary export format starts with a 'c', 'd', or 'v'
// (from "version"). Select appropriate importer.
if len(data) > 0 && data[0] == 'i' {
_, pkg, err := gcimporter.IImportData(fset, imports, data[1:], path)
return pkg, err
}
_, pkg, err := gcimporter.BImportData(fset, imports, data, path) _, pkg, err := gcimporter.BImportData(fset, imports, data, path)
return pkg, err return pkg, err
} }
...@@ -95,6 +100,10 @@ func Read(in io.Reader, fset *token.FileSet, imports map[string]*types.Package, ...@@ -95,6 +100,10 @@ func Read(in io.Reader, fset *token.FileSet, imports map[string]*types.Package,
// Write writes encoded type information for the specified package to out. // Write writes encoded type information for the specified package to out.
// The FileSet provides file position information for named objects. // The FileSet provides file position information for named objects.
func Write(out io.Writer, fset *token.FileSet, pkg *types.Package) error { func Write(out io.Writer, fset *token.FileSet, pkg *types.Package) error {
_, err := out.Write(gcimporter.BExportData(fset, pkg)) b, err := gcimporter.BExportData(fset, pkg)
if err != nil {
return err
}
_, err = out.Write(b)
return err return err
} }
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build ignore
// The gcexportdata command is a diagnostic tool that displays the
// contents of gc export data files.
package main
import (
"flag"
"fmt"
"go/token"
"go/types"
"log"
"os"
"golang.org/x/tools/go/gcexportdata"
"golang.org/x/tools/go/types/typeutil"
)
var packageFlag = flag.String("package", "", "alternative package to print")
func main() {
log.SetPrefix("gcexportdata: ")
log.SetFlags(0)
flag.Usage = func() {
fmt.Fprintln(os.Stderr, "usage: gcexportdata [-package path] file.a")
}
flag.Parse()
if flag.NArg() != 1 {
flag.Usage()
os.Exit(2)
}
filename := flag.Args()[0]
f, err := os.Open(filename)
if err != nil {
log.Fatal(err)
}
r, err := gcexportdata.NewReader(f)
if err != nil {
log.Fatalf("%s: %s", filename, err)
}
// Decode the package.
const primary = "<primary>"
imports := make(map[string]*types.Package)
fset := token.NewFileSet()
pkg, err := gcexportdata.Read(r, fset, imports, primary)
if err != nil {
log.Fatalf("%s: %s", filename, err)
}
// Optionally select an indirectly mentioned package.
if *packageFlag != "" {
pkg = imports[*packageFlag]
if pkg == nil {
fmt.Fprintf(os.Stderr, "export data file %s does not mention %s; has:\n",
filename, *packageFlag)
for p := range imports {
if p != primary {
fmt.Fprintf(os.Stderr, "\t%s\n", p)
}
}
os.Exit(1)
}
}
// Print all package-level declarations, including non-exported ones.
fmt.Printf("package %s\n", pkg.Name())
for _, imp := range pkg.Imports() {
fmt.Printf("import %q\n", imp.Path())
}
qual := func(p *types.Package) string {
if pkg == p {
return ""
}
return p.Name()
}
scope := pkg.Scope()
for _, name := range scope.Names() {
obj := scope.Lookup(name)
fmt.Printf("%s: %s\n",
fset.Position(obj.Pos()),
types.ObjectString(obj, qual))
// For types, print each method.
if _, ok := obj.(*types.TypeName); ok {
for _, method := range typeutil.IntuitiveMethodSet(obj.Type(), nil) {
fmt.Printf("%s: %s\n",
fset.Position(method.Obj().Pos()),
types.SelectionString(method, qual))
}
}
}
}
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build go1.9
package gcimporter
import "go/types"
func isAlias(obj *types.TypeName) bool {
return obj.IsAlias()
}
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"cgo.go",
"cgo_pkgconfig.go",
],
importmap = "k8s.io/kubernetes/vendor/golang.org/x/tools/go/internal/cgo",
importpath = "golang.org/x/tools/go/internal/cgo",
visibility = ["//vendor/golang.org/x/tools/go:__subpackages__"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cgo
// This file handles cgo preprocessing of files containing `import "C"`.
//
// DESIGN
//
// The approach taken is to run the cgo processor on the package's
// CgoFiles and parse the output, faking the filenames of the
// resulting ASTs so that the synthetic file containing the C types is
// called "C" (e.g. "~/go/src/net/C") and the preprocessed files
// have their original names (e.g. "~/go/src/net/cgo_unix.go"),
// not the names of the actual temporary files.
//
// The advantage of this approach is its fidelity to 'go build'. The
// downside is that the token.Position.Offset for each AST node is
// incorrect, being an offset within the temporary file. Line numbers
// should still be correct because of the //line comments.
//
// The logic of this file is mostly plundered from the 'go build'
// tool, which also invokes the cgo preprocessor.
//
//
// REJECTED ALTERNATIVE
//
// An alternative approach that we explored is to extend go/types'
// Importer mechanism to provide the identity of the importing package
// so that each time `import "C"` appears it resolves to a different
// synthetic package containing just the objects needed in that case.
// The loader would invoke cgo but parse only the cgo_types.go file
// defining the package-level objects, discarding the other files
// resulting from preprocessing.
//
// The benefit of this approach would have been that source-level
// syntax information would correspond exactly to the original cgo
// file, with no preprocessing involved, making source tools like
// godoc, guru, and eg happy. However, the approach was rejected
// due to the additional complexity it would impose on go/types. (It
// made for a beautiful demo, though.)
//
// cgo files, despite their *.go extension, are not legal Go source
// files per the specification since they may refer to unexported
// members of package "C" such as C.int. Also, a function such as
// C.getpwent has in effect two types, one matching its C type and one
// which additionally returns (errno C.int). The cgo preprocessor
// uses name mangling to distinguish these two functions in the
// processed code, but go/types would need to duplicate this logic in
// its handling of function calls, analogous to the treatment of map
// lookups in which y=m[k] and y,ok=m[k] are both legal.
import (
"fmt"
"go/ast"
"go/build"
"go/parser"
"go/token"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
)
// ProcessFiles invokes the cgo preprocessor on bp.CgoFiles, parses
// the output and returns the resulting ASTs.
//
func ProcessFiles(bp *build.Package, fset *token.FileSet, DisplayPath func(path string) string, mode parser.Mode) ([]*ast.File, error) {
tmpdir, err := ioutil.TempDir("", strings.Replace(bp.ImportPath, "/", "_", -1)+"_C")
if err != nil {
return nil, err
}
defer os.RemoveAll(tmpdir)
pkgdir := bp.Dir
if DisplayPath != nil {
pkgdir = DisplayPath(pkgdir)
}
cgoFiles, cgoDisplayFiles, err := Run(bp, pkgdir, tmpdir, false)
if err != nil {
return nil, err
}
var files []*ast.File
for i := range cgoFiles {
rd, err := os.Open(cgoFiles[i])
if err != nil {
return nil, err
}
display := filepath.Join(bp.Dir, cgoDisplayFiles[i])
f, err := parser.ParseFile(fset, display, rd, mode)
rd.Close()
if err != nil {
return nil, err
}
files = append(files, f)
}
return files, nil
}
var cgoRe = regexp.MustCompile(`[/\\:]`)
// Run invokes the cgo preprocessor on bp.CgoFiles and returns two
// lists of files: the resulting processed files (in temporary
// directory tmpdir) and the corresponding names of the unprocessed files.
//
// Run is adapted from (*builder).cgo in
// $GOROOT/src/cmd/go/build.go, but these features are unsupported:
// Objective C, CGOPKGPATH, CGO_FLAGS.
//
// If useabs is set to true, absolute paths of the bp.CgoFiles will be passed in
// to the cgo preprocessor. This in turn will set the // line comments
// referring to those files to use absolute paths. This is needed for
// go/packages using the legacy go list support so it is able to find
// the original files.
func Run(bp *build.Package, pkgdir, tmpdir string, useabs bool) (files, displayFiles []string, err error) {
cgoCPPFLAGS, _, _, _ := cflags(bp, true)
_, cgoexeCFLAGS, _, _ := cflags(bp, false)
if len(bp.CgoPkgConfig) > 0 {
pcCFLAGS, err := pkgConfigFlags(bp)
if err != nil {
return nil, nil, err
}
cgoCPPFLAGS = append(cgoCPPFLAGS, pcCFLAGS...)
}
// Allows including _cgo_export.h from .[ch] files in the package.
cgoCPPFLAGS = append(cgoCPPFLAGS, "-I", tmpdir)
// _cgo_gotypes.go (displayed "C") contains the type definitions.
files = append(files, filepath.Join(tmpdir, "_cgo_gotypes.go"))
displayFiles = append(displayFiles, "C")
for _, fn := range bp.CgoFiles {
// "foo.cgo1.go" (displayed "foo.go") is the processed Go source.
f := cgoRe.ReplaceAllString(fn[:len(fn)-len("go")], "_")
files = append(files, filepath.Join(tmpdir, f+"cgo1.go"))
displayFiles = append(displayFiles, fn)
}
var cgoflags []string
if bp.Goroot && bp.ImportPath == "runtime/cgo" {
cgoflags = append(cgoflags, "-import_runtime_cgo=false")
}
if bp.Goroot && bp.ImportPath == "runtime/race" || bp.ImportPath == "runtime/cgo" {
cgoflags = append(cgoflags, "-import_syscall=false")
}
var cgoFiles []string = bp.CgoFiles
if useabs {
cgoFiles = make([]string, len(bp.CgoFiles))
for i := range cgoFiles {
cgoFiles[i] = filepath.Join(pkgdir, bp.CgoFiles[i])
}
}
args := stringList(
"go", "tool", "cgo", "-objdir", tmpdir, cgoflags, "--",
cgoCPPFLAGS, cgoexeCFLAGS, cgoFiles,
)
if false {
log.Printf("Running cgo for package %q: %s (dir=%s)", bp.ImportPath, args, pkgdir)
}
cmd := exec.Command(args[0], args[1:]...)
cmd.Dir = pkgdir
cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return nil, nil, fmt.Errorf("cgo failed: %s: %s", args, err)
}
return files, displayFiles, nil
}
// -- unmodified from 'go build' ---------------------------------------
// Return the flags to use when invoking the C or C++ compilers, or cgo.
func cflags(p *build.Package, def bool) (cppflags, cflags, cxxflags, ldflags []string) {
var defaults string
if def {
defaults = "-g -O2"
}
cppflags = stringList(envList("CGO_CPPFLAGS", ""), p.CgoCPPFLAGS)
cflags = stringList(envList("CGO_CFLAGS", defaults), p.CgoCFLAGS)
cxxflags = stringList(envList("CGO_CXXFLAGS", defaults), p.CgoCXXFLAGS)
ldflags = stringList(envList("CGO_LDFLAGS", defaults), p.CgoLDFLAGS)
return
}
// envList returns the value of the given environment variable broken
// into fields, using the default value when the variable is empty.
func envList(key, def string) []string {
v := os.Getenv(key)
if v == "" {
v = def
}
return strings.Fields(v)
}
// stringList's arguments should be a sequence of string or []string values.
// stringList flattens them into a single []string.
func stringList(args ...interface{}) []string {
var x []string
for _, arg := range args {
switch arg := arg.(type) {
case []string:
x = append(x, arg...)
case string:
x = append(x, arg)
default:
panic("stringList: invalid argument")
}
}
return x
}
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cgo
import (
"errors"
"fmt"
"go/build"
"os/exec"
"strings"
)
// pkgConfig runs pkg-config with the specified arguments and returns the flags it prints.
func pkgConfig(mode string, pkgs []string) (flags []string, err error) {
cmd := exec.Command("pkg-config", append([]string{mode}, pkgs...)...)
out, err := cmd.CombinedOutput()
if err != nil {
s := fmt.Sprintf("%s failed: %v", strings.Join(cmd.Args, " "), err)
if len(out) > 0 {
s = fmt.Sprintf("%s: %s", s, out)
}
return nil, errors.New(s)
}
if len(out) > 0 {
flags = strings.Fields(string(out))
}
return
}
// pkgConfigFlags calls pkg-config if needed and returns the cflags
// needed to build the package.
func pkgConfigFlags(p *build.Package) (cflags []string, err error) {
if len(p.CgoPkgConfig) == 0 {
return nil, nil
}
return pkgConfig("--cflags", p.CgoPkgConfig)
}
...@@ -7,12 +7,14 @@ go_library( ...@@ -7,12 +7,14 @@ go_library(
"bimport.go", "bimport.go",
"exportdata.go", "exportdata.go",
"gcimporter.go", "gcimporter.go",
"isAlias18.go", "iexport.go",
"isAlias19.go", "iimport.go",
"newInterface10.go",
"newInterface11.go",
], ],
importmap = "k8s.io/kubernetes/vendor/golang.org/x/tools/go/gcimporter15", importmap = "k8s.io/kubernetes/vendor/golang.org/x/tools/go/internal/gcimporter",
importpath = "golang.org/x/tools/go/gcimporter15", importpath = "golang.org/x/tools/go/internal/gcimporter",
visibility = ["//visibility:public"], visibility = ["//vendor/golang.org/x/tools/go:__subpackages__"],
) )
filegroup( filegroup(
......
...@@ -16,7 +16,6 @@ import ( ...@@ -16,7 +16,6 @@ import (
"go/constant" "go/constant"
"go/token" "go/token"
"go/types" "go/types"
"log"
"math" "math"
"math/big" "math/big"
"sort" "sort"
...@@ -39,6 +38,11 @@ const debugFormat = false // default: false ...@@ -39,6 +38,11 @@ const debugFormat = false // default: false
const trace = false // default: false const trace = false // default: false
// Current export format version. Increase with each format change. // Current export format version. Increase with each format change.
// Note: The latest binary (non-indexed) export format is at version 6.
// This exporter is still at level 4, but it doesn't matter since
// the binary importer can handle older versions just fine.
// 6: package height (CL 105038) -- NOT IMPLEMENTED HERE
// 5: improved position encoding efficiency (issue 20080, CL 41619) -- NOT IMPLEMEMTED HERE
// 4: type name objects support type aliases, uses aliasTag // 4: type name objects support type aliases, uses aliasTag
// 3: Go1.8 encoding (same as version 2, aliasTag defined but never used) // 3: Go1.8 encoding (same as version 2, aliasTag defined but never used)
// 2: removed unused bool in ODCL export (compiler only) // 2: removed unused bool in ODCL export (compiler only)
...@@ -76,9 +80,29 @@ type exporter struct { ...@@ -76,9 +80,29 @@ type exporter struct {
indent int // for trace indent int // for trace
} }
// internalError represents an error generated inside this package.
type internalError string
func (e internalError) Error() string { return "gcimporter: " + string(e) }
func internalErrorf(format string, args ...interface{}) error {
return internalError(fmt.Sprintf(format, args...))
}
// BExportData returns binary export data for pkg. // BExportData returns binary export data for pkg.
// If no file set is provided, position info will be missing. // If no file set is provided, position info will be missing.
func BExportData(fset *token.FileSet, pkg *types.Package) []byte { func BExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error) {
defer func() {
if e := recover(); e != nil {
if ierr, ok := e.(internalError); ok {
err = ierr
return
}
// Not an internal error; panic again.
panic(e)
}
}()
p := exporter{ p := exporter{
fset: fset, fset: fset,
strIndex: map[string]int{"": 0}, // empty string is mapped to 0 strIndex: map[string]int{"": 0}, // empty string is mapped to 0
...@@ -103,11 +127,11 @@ func BExportData(fset *token.FileSet, pkg *types.Package) []byte { ...@@ -103,11 +127,11 @@ func BExportData(fset *token.FileSet, pkg *types.Package) []byte {
// --- generic export data --- // --- generic export data ---
// populate type map with predeclared "known" types // populate type map with predeclared "known" types
for index, typ := range predeclared { for index, typ := range predeclared() {
p.typIndex[typ] = index p.typIndex[typ] = index
} }
if len(p.typIndex) != len(predeclared) { if len(p.typIndex) != len(predeclared()) {
log.Fatalf("gcimporter: duplicate entries in type map?") return nil, internalError("duplicate entries in type map?")
} }
// write package data // write package data
...@@ -145,12 +169,12 @@ func BExportData(fset *token.FileSet, pkg *types.Package) []byte { ...@@ -145,12 +169,12 @@ func BExportData(fset *token.FileSet, pkg *types.Package) []byte {
// --- end of export data --- // --- end of export data ---
return p.out.Bytes() return p.out.Bytes(), nil
} }
func (p *exporter) pkg(pkg *types.Package, emptypath bool) { func (p *exporter) pkg(pkg *types.Package, emptypath bool) {
if pkg == nil { if pkg == nil {
log.Fatalf("gcimporter: unexpected nil pkg") panic(internalError("unexpected nil pkg"))
} }
// if we saw the package before, write its index (>= 0) // if we saw the package before, write its index (>= 0)
...@@ -185,7 +209,7 @@ func (p *exporter) obj(obj types.Object) { ...@@ -185,7 +209,7 @@ func (p *exporter) obj(obj types.Object) {
p.value(obj.Val()) p.value(obj.Val())
case *types.TypeName: case *types.TypeName:
if isAlias(obj) { if obj.IsAlias() {
p.tag(aliasTag) p.tag(aliasTag)
p.pos(obj) p.pos(obj)
p.qualifiedName(obj) p.qualifiedName(obj)
...@@ -209,7 +233,7 @@ func (p *exporter) obj(obj types.Object) { ...@@ -209,7 +233,7 @@ func (p *exporter) obj(obj types.Object) {
p.paramList(sig.Results(), false) p.paramList(sig.Results(), false)
default: default:
log.Fatalf("gcimporter: unexpected object %v (%T)", obj, obj) panic(internalErrorf("unexpected object %v (%T)", obj, obj))
} }
} }
...@@ -273,7 +297,7 @@ func (p *exporter) qualifiedName(obj types.Object) { ...@@ -273,7 +297,7 @@ func (p *exporter) qualifiedName(obj types.Object) {
func (p *exporter) typ(t types.Type) { func (p *exporter) typ(t types.Type) {
if t == nil { if t == nil {
log.Fatalf("gcimporter: nil type") panic(internalError("nil type"))
} }
// Possible optimization: Anonymous pointer types *T where // Possible optimization: Anonymous pointer types *T where
...@@ -356,7 +380,7 @@ func (p *exporter) typ(t types.Type) { ...@@ -356,7 +380,7 @@ func (p *exporter) typ(t types.Type) {
p.typ(t.Elem()) p.typ(t.Elem())
default: default:
log.Fatalf("gcimporter: unexpected type %T: %s", t, t) panic(internalErrorf("unexpected type %T: %s", t, t))
} }
} }
...@@ -422,7 +446,7 @@ func (p *exporter) fieldList(t *types.Struct) { ...@@ -422,7 +446,7 @@ func (p *exporter) fieldList(t *types.Struct) {
func (p *exporter) field(f *types.Var) { func (p *exporter) field(f *types.Var) {
if !f.IsField() { if !f.IsField() {
log.Fatalf("gcimporter: field expected") panic(internalError("field expected"))
} }
p.pos(f) p.pos(f)
...@@ -452,7 +476,7 @@ func (p *exporter) iface(t *types.Interface) { ...@@ -452,7 +476,7 @@ func (p *exporter) iface(t *types.Interface) {
func (p *exporter) method(m *types.Func) { func (p *exporter) method(m *types.Func) {
sig := m.Type().(*types.Signature) sig := m.Type().(*types.Signature)
if sig.Recv() == nil { if sig.Recv() == nil {
log.Fatalf("gcimporter: method expected") panic(internalError("method expected"))
} }
p.pos(m) p.pos(m)
...@@ -575,13 +599,13 @@ func (p *exporter) value(x constant.Value) { ...@@ -575,13 +599,13 @@ func (p *exporter) value(x constant.Value) {
p.tag(unknownTag) p.tag(unknownTag)
default: default:
log.Fatalf("gcimporter: unexpected value %v (%T)", x, x) panic(internalErrorf("unexpected value %v (%T)", x, x))
} }
} }
func (p *exporter) float(x constant.Value) { func (p *exporter) float(x constant.Value) {
if x.Kind() != constant.Float { if x.Kind() != constant.Float {
log.Fatalf("gcimporter: unexpected constant %v, want float", x) panic(internalErrorf("unexpected constant %v, want float", x))
} }
// extract sign (there is no -0) // extract sign (there is no -0)
sign := constant.Sign(x) sign := constant.Sign(x)
...@@ -616,7 +640,7 @@ func (p *exporter) float(x constant.Value) { ...@@ -616,7 +640,7 @@ func (p *exporter) float(x constant.Value) {
m.SetMantExp(&m, int(m.MinPrec())) m.SetMantExp(&m, int(m.MinPrec()))
mant, acc := m.Int(nil) mant, acc := m.Int(nil)
if acc != big.Exact { if acc != big.Exact {
log.Fatalf("gcimporter: internal error") panic(internalError("internal error"))
} }
p.int(sign) p.int(sign)
...@@ -653,7 +677,7 @@ func (p *exporter) bool(b bool) bool { ...@@ -653,7 +677,7 @@ func (p *exporter) bool(b bool) bool {
func (p *exporter) index(marker byte, index int) { func (p *exporter) index(marker byte, index int) {
if index < 0 { if index < 0 {
log.Fatalf("gcimporter: invalid index < 0") panic(internalError("invalid index < 0"))
} }
if debugFormat { if debugFormat {
p.marker('t') p.marker('t')
...@@ -666,7 +690,7 @@ func (p *exporter) index(marker byte, index int) { ...@@ -666,7 +690,7 @@ func (p *exporter) index(marker byte, index int) {
func (p *exporter) tag(tag int) { func (p *exporter) tag(tag int) {
if tag >= 0 { if tag >= 0 {
log.Fatalf("gcimporter: invalid tag >= 0") panic(internalError("invalid tag >= 0"))
} }
if debugFormat { if debugFormat {
p.marker('t') p.marker('t')
......
...@@ -2,17 +2,13 @@ ...@@ -2,17 +2,13 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
// This file is a copy of $GOROOT/src/go/internal/gcimporter/gcimporter.go, // This file is a modified copy of $GOROOT/src/go/internal/gcimporter/gcimporter.go,
// but it also contains the original source-based importer code for Go1.6. // but it also contains the original source-based importer code for Go1.6.
// Once we stop supporting 1.6, we can remove that code. // Once we stop supporting 1.6, we can remove that code.
// Package gcimporter15 provides various functions for reading // Package gcimporter provides various functions for reading
// gc-generated object files that can be used to implement the // gc-generated object files that can be used to implement the
// Importer interface defined by the Go 1.5 standard library package. // Importer interface defined by the Go 1.5 standard library package.
//
// Deprecated: this package will be deleted in October 2017.
// New code should use golang.org/x/tools/go/gcexportdata.
//
package gcimporter package gcimporter
import ( import (
...@@ -20,7 +16,7 @@ import ( ...@@ -20,7 +16,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"go/build" "go/build"
exact "go/constant" "go/constant"
"go/token" "go/token"
"go/types" "go/types"
"io" "io"
...@@ -59,6 +55,7 @@ func FindPkg(path, srcDir string) (filename, id string) { ...@@ -59,6 +55,7 @@ func FindPkg(path, srcDir string) (filename, id string) {
} }
bp, _ := build.Import(path, srcDir, build.FindOnly|build.AllowBinary) bp, _ := build.Import(path, srcDir, build.FindOnly|build.AllowBinary)
if bp.PkgObj == "" { if bp.PkgObj == "" {
id = path // make sure we have an id to print in error message
return return
} }
noext = strings.TrimSuffix(bp.PkgObj, ".a") noext = strings.TrimSuffix(bp.PkgObj, ".a")
...@@ -131,51 +128,91 @@ func ImportData(packages map[string]*types.Package, filename, id string, data io ...@@ -131,51 +128,91 @@ func ImportData(packages map[string]*types.Package, filename, id string, data io
// the corresponding package object to the packages map, and returns the object. // the corresponding package object to the packages map, and returns the object.
// The packages map must contain all packages already imported. // The packages map must contain all packages already imported.
// //
func Import(packages map[string]*types.Package, path, srcDir string) (pkg *types.Package, err error) { func Import(packages map[string]*types.Package, path, srcDir string, lookup func(path string) (io.ReadCloser, error)) (pkg *types.Package, err error) {
filename, id := FindPkg(path, srcDir) var rc io.ReadCloser
if filename == "" { var filename, id string
if lookup != nil {
// With custom lookup specified, assume that caller has
// converted path to a canonical import path for use in the map.
if path == "unsafe" { if path == "unsafe" {
return types.Unsafe, nil return types.Unsafe, nil
} }
err = fmt.Errorf("can't find import: %s", id) id = path
return
}
// no need to re-import if the package was imported completely before // No need to re-import if the package was imported completely before.
if pkg = packages[id]; pkg != nil && pkg.Complete() { if pkg = packages[id]; pkg != nil && pkg.Complete() {
return return
} }
f, err := lookup(path)
if err != nil {
return nil, err
}
rc = f
} else {
filename, id = FindPkg(path, srcDir)
if filename == "" {
if path == "unsafe" {
return types.Unsafe, nil
}
return nil, fmt.Errorf("can't find import: %q", id)
}
// open file // no need to re-import if the package was imported completely before
f, err := os.Open(filename) if pkg = packages[id]; pkg != nil && pkg.Complete() {
if err != nil { return
return }
}
defer func() { // open file
f.Close() f, err := os.Open(filename)
if err != nil { if err != nil {
// add file name to error return nil, err
err = fmt.Errorf("reading export data: %s: %v", filename, err)
} }
}() defer func() {
if err != nil {
// add file name to error
err = fmt.Errorf("%s: %v", filename, err)
}
}()
rc = f
}
defer rc.Close()
var hdr string var hdr string
buf := bufio.NewReader(f) buf := bufio.NewReader(rc)
if hdr, err = FindExportData(buf); err != nil { if hdr, err = FindExportData(buf); err != nil {
return return
} }
switch hdr { switch hdr {
case "$$\n": case "$$\n":
// Work-around if we don't have a filename; happens only if lookup != nil.
// Either way, the filename is only needed for importer error messages, so
// this is fine.
if filename == "" {
filename = path
}
return ImportData(packages, filename, id, buf) return ImportData(packages, filename, id, buf)
case "$$B\n": case "$$B\n":
var data []byte var data []byte
data, err = ioutil.ReadAll(buf) data, err = ioutil.ReadAll(buf)
if err == nil { if err != nil {
fset := token.NewFileSet() break
}
// TODO(gri): allow clients of go/importer to provide a FileSet.
// Or, define a new standard go/types/gcexportdata package.
fset := token.NewFileSet()
// The indexed export format starts with an 'i'; the older
// binary export format starts with a 'c', 'd', or 'v'
// (from "version"). Select appropriate importer.
if len(data) > 0 && data[0] == 'i' {
_, pkg, err = IImportData(fset, packages, data[1:], id)
} else {
_, pkg, err = BImportData(fset, packages, data, id) _, pkg, err = BImportData(fset, packages, data, id)
return
} }
default: default:
err = fmt.Errorf("unknown export data header: %q", hdr) err = fmt.Errorf("unknown export data header: %q", hdr)
} }
...@@ -767,9 +804,9 @@ func (p *parser) parseInt() string { ...@@ -767,9 +804,9 @@ func (p *parser) parseInt() string {
// number = int_lit [ "p" int_lit ] . // number = int_lit [ "p" int_lit ] .
// //
func (p *parser) parseNumber() (typ *types.Basic, val exact.Value) { func (p *parser) parseNumber() (typ *types.Basic, val constant.Value) {
// mantissa // mantissa
mant := exact.MakeFromLiteral(p.parseInt(), token.INT, 0) mant := constant.MakeFromLiteral(p.parseInt(), token.INT, 0)
if mant == nil { if mant == nil {
panic("invalid mantissa") panic("invalid mantissa")
} }
...@@ -782,14 +819,14 @@ func (p *parser) parseNumber() (typ *types.Basic, val exact.Value) { ...@@ -782,14 +819,14 @@ func (p *parser) parseNumber() (typ *types.Basic, val exact.Value) {
p.error(err) p.error(err)
} }
if exp < 0 { if exp < 0 {
denom := exact.MakeInt64(1) denom := constant.MakeInt64(1)
denom = exact.Shift(denom, token.SHL, uint(-exp)) denom = constant.Shift(denom, token.SHL, uint(-exp))
typ = types.Typ[types.UntypedFloat] typ = types.Typ[types.UntypedFloat]
val = exact.BinaryOp(mant, token.QUO, denom) val = constant.BinaryOp(mant, token.QUO, denom)
return return
} }
if exp > 0 { if exp > 0 {
mant = exact.Shift(mant, token.SHL, uint(exp)) mant = constant.Shift(mant, token.SHL, uint(exp))
} }
typ = types.Typ[types.UntypedFloat] typ = types.Typ[types.UntypedFloat]
val = mant val = mant
...@@ -820,7 +857,7 @@ func (p *parser) parseConstDecl() { ...@@ -820,7 +857,7 @@ func (p *parser) parseConstDecl() {
p.expect('=') p.expect('=')
var typ types.Type var typ types.Type
var val exact.Value var val constant.Value
switch p.tok { switch p.tok {
case scanner.Ident: case scanner.Ident:
// bool_lit // bool_lit
...@@ -828,7 +865,7 @@ func (p *parser) parseConstDecl() { ...@@ -828,7 +865,7 @@ func (p *parser) parseConstDecl() {
p.error("expected true or false") p.error("expected true or false")
} }
typ = types.Typ[types.UntypedBool] typ = types.Typ[types.UntypedBool]
val = exact.MakeBool(p.lit == "true") val = constant.MakeBool(p.lit == "true")
p.next() p.next()
case '-', scanner.Int: case '-', scanner.Int:
...@@ -852,18 +889,18 @@ func (p *parser) parseConstDecl() { ...@@ -852,18 +889,18 @@ func (p *parser) parseConstDecl() {
p.expectKeyword("i") p.expectKeyword("i")
p.expect(')') p.expect(')')
typ = types.Typ[types.UntypedComplex] typ = types.Typ[types.UntypedComplex]
val = exact.BinaryOp(re, token.ADD, exact.MakeImag(im)) val = constant.BinaryOp(re, token.ADD, constant.MakeImag(im))
case scanner.Char: case scanner.Char:
// rune_lit // rune_lit
typ = types.Typ[types.UntypedRune] typ = types.Typ[types.UntypedRune]
val = exact.MakeFromLiteral(p.lit, token.CHAR, 0) val = constant.MakeFromLiteral(p.lit, token.CHAR, 0)
p.next() p.next()
case scanner.String: case scanner.String:
// string_lit // string_lit
typ = types.Typ[types.UntypedString] typ = types.Typ[types.UntypedString]
val = exact.MakeFromLiteral(p.lit, token.STRING, 0) val = constant.MakeFromLiteral(p.lit, token.STRING, 0)
p.next() p.next()
default: default:
......
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !go1.11
package gcimporter
import "go/types"
func newInterface(methods []*types.Func, embeddeds []types.Type) *types.Interface {
named := make([]*types.Named, len(embeddeds))
for i, e := range embeddeds {
var ok bool
named[i], ok = e.(*types.Named)
if !ok {
panic("embedding of non-defined interfaces in interfaces is not supported before Go 1.11")
}
}
return types.NewInterface(methods, named)
}
// Copyright 2017 The Go Authors. All rights reserved. // Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
// +build !go1.9 // +build go1.11
package gcimporter package gcimporter
import "go/types" import "go/types"
func isAlias(obj *types.TypeName) bool { func newInterface(methods []*types.Func, embeddeds []types.Type) *types.Interface {
return false // there are no type aliases before Go 1.9 return types.NewInterfaceType(methods, embeddeds)
} }
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["sizes.go"],
importmap = "k8s.io/kubernetes/vendor/golang.org/x/tools/go/internal/packagesdriver",
importpath = "golang.org/x/tools/go/internal/packagesdriver",
visibility = ["//vendor/golang.org/x/tools/go:__subpackages__"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package packagesdriver fetches type sizes for go/packages and go/analysis.
package packagesdriver
import (
"bytes"
"context"
"encoding/json"
"fmt"
"go/types"
"log"
"os"
"os/exec"
"strings"
"time"
)
var debug = false
// GetSizes returns the sizes used by the underlying driver with the given parameters.
func GetSizes(ctx context.Context, buildFlags, env []string, dir string, usesExportData bool) (types.Sizes, error) {
// TODO(matloob): Clean this up. This code is mostly a copy of packages.findExternalDriver.
const toolPrefix = "GOPACKAGESDRIVER="
tool := ""
for _, env := range env {
if val := strings.TrimPrefix(env, toolPrefix); val != env {
tool = val
}
}
if tool == "" {
var err error
tool, err = exec.LookPath("gopackagesdriver")
if err != nil {
// We did not find the driver, so use "go list".
tool = "off"
}
}
if tool == "off" {
return GetSizesGolist(ctx, buildFlags, env, dir, usesExportData)
}
req, err := json.Marshal(struct {
Command string `json:"command"`
Env []string `json:"env"`
BuildFlags []string `json:"build_flags"`
}{
Command: "sizes",
Env: env,
BuildFlags: buildFlags,
})
if err != nil {
return nil, fmt.Errorf("failed to encode message to driver tool: %v", err)
}
buf := new(bytes.Buffer)
cmd := exec.CommandContext(ctx, tool)
cmd.Dir = dir
cmd.Env = env
cmd.Stdin = bytes.NewReader(req)
cmd.Stdout = buf
cmd.Stderr = new(bytes.Buffer)
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("%v: %v: %s", tool, err, cmd.Stderr)
}
var response struct {
// Sizes, if not nil, is the types.Sizes to use when type checking.
Sizes *types.StdSizes
}
if err := json.Unmarshal(buf.Bytes(), &response); err != nil {
return nil, err
}
return response.Sizes, nil
}
func GetSizesGolist(ctx context.Context, buildFlags, env []string, dir string, usesExportData bool) (types.Sizes, error) {
args := []string{"list", "-f", "{{context.GOARCH}} {{context.Compiler}}"}
args = append(args, buildFlags...)
args = append(args, "--", "unsafe")
stdout, err := InvokeGo(ctx, env, dir, usesExportData, args...)
if err != nil {
return nil, err
}
fields := strings.Fields(stdout.String())
if len(fields) < 2 {
return nil, fmt.Errorf("could not determine GOARCH and Go compiler")
}
goarch := fields[0]
compiler := fields[1]
return types.SizesFor(compiler, goarch), nil
}
// InvokeGo returns the stdout of a go command invocation.
func InvokeGo(ctx context.Context, env []string, dir string, usesExportData bool, args ...string) (*bytes.Buffer, error) {
if debug {
defer func(start time.Time) { log.Printf("%s for %v", time.Since(start), cmdDebugStr(env, args...)) }(time.Now())
}
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
cmd := exec.CommandContext(ctx, "go", args...)
// On darwin the cwd gets resolved to the real path, which breaks anything that
// expects the working directory to keep the original path, including the
// go command when dealing with modules.
// The Go stdlib has a special feature where if the cwd and the PWD are the
// same node then it trusts the PWD, so by setting it in the env for the child
// process we fix up all the paths returned by the go command.
cmd.Env = append(append([]string{}, env...), "PWD="+dir)
cmd.Dir = dir
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
exitErr, ok := err.(*exec.ExitError)
if !ok {
// Catastrophic error:
// - executable not found
// - context cancellation
return nil, fmt.Errorf("couldn't exec 'go %v': %s %T", args, err, err)
}
// Export mode entails a build.
// If that build fails, errors appear on stderr
// (despite the -e flag) and the Export field is blank.
// Do not fail in that case.
if !usesExportData {
return nil, fmt.Errorf("go %v: %s: %s", args, exitErr, stderr)
}
}
// As of writing, go list -export prints some non-fatal compilation
// errors to stderr, even with -e set. We would prefer that it put
// them in the Package.Error JSON (see https://golang.org/issue/26319).
// In the meantime, there's nowhere good to put them, but they can
// be useful for debugging. Print them if $GOPACKAGESPRINTGOLISTERRORS
// is set.
if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTGOLISTERRORS") != "" {
fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(env, args...), stderr)
}
// debugging
if false {
fmt.Fprintf(os.Stderr, "%s stdout: <<%s>>\n", cmdDebugStr(env, args...), stdout)
}
return stdout, nil
}
func cmdDebugStr(envlist []string, args ...string) string {
env := make(map[string]string)
for _, kv := range envlist {
split := strings.Split(kv, "=")
k, v := split[0], split[1]
env[k] = v
}
return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v PWD=%v go %v", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["PWD"], args)
}
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"doc.go",
"external.go",
"golist.go",
"golist_fallback.go",
"golist_fallback_testmain.go",
"golist_overlay.go",
"packages.go",
"visit.go",
],
importmap = "k8s.io/kubernetes/vendor/golang.org/x/tools/go/packages",
importpath = "golang.org/x/tools/go/packages",
visibility = ["//visibility:public"],
deps = [
"//vendor/golang.org/x/tools/go/gcexportdata:go_default_library",
"//vendor/golang.org/x/tools/go/internal/cgo:go_default_library",
"//vendor/golang.org/x/tools/go/internal/packagesdriver:go_default_library",
"//vendor/golang.org/x/tools/internal/gopathwalk:go_default_library",
"//vendor/golang.org/x/tools/internal/semver:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// This file enables an external tool to intercept package requests.
// If the tool is present then its results are used in preference to
// the go list command.
package packages
import (
"bytes"
"encoding/json"
"fmt"
"os/exec"
"strings"
)
// Driver
type driverRequest struct {
Command string `json "command"`
Mode LoadMode `json:"mode"`
Env []string `json:"env"`
BuildFlags []string `json:"build_flags"`
Tests bool `json:"tests"`
Overlay map[string][]byte `json:"overlay"`
}
// findExternalDriver returns the file path of a tool that supplies
// the build system package structure, or "" if not found."
// If GOPACKAGESDRIVER is set in the environment findExternalTool returns its
// value, otherwise it searches for a binary named gopackagesdriver on the PATH.
func findExternalDriver(cfg *Config) driver {
const toolPrefix = "GOPACKAGESDRIVER="
tool := ""
for _, env := range cfg.Env {
if val := strings.TrimPrefix(env, toolPrefix); val != env {
tool = val
}
}
if tool != "" && tool == "off" {
return nil
}
if tool == "" {
var err error
tool, err = exec.LookPath("gopackagesdriver")
if err != nil {
return nil
}
}
return func(cfg *Config, words ...string) (*driverResponse, error) {
req, err := json.Marshal(driverRequest{
Mode: cfg.Mode,
Env: cfg.Env,
BuildFlags: cfg.BuildFlags,
Tests: cfg.Tests,
Overlay: cfg.Overlay,
})
if err != nil {
return nil, fmt.Errorf("failed to encode message to driver tool: %v", err)
}
buf := new(bytes.Buffer)
cmd := exec.CommandContext(cfg.Context, tool, words...)
cmd.Dir = cfg.Dir
cmd.Env = cfg.Env
cmd.Stdin = bytes.NewReader(req)
cmd.Stdout = buf
cmd.Stderr = new(bytes.Buffer)
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("%v: %v: %s", tool, err, cmd.Stderr)
}
var response driverResponse
if err := json.Unmarshal(buf.Bytes(), &response); err != nil {
return nil, err
}
return &response, nil
}
}
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// This file is largely based on the Go 1.10-era cmd/go/internal/test/test.go
// testmain generation code.
package packages
import (
"errors"
"fmt"
"go/ast"
"go/doc"
"go/parser"
"go/token"
"os"
"sort"
"strings"
"text/template"
"unicode"
"unicode/utf8"
)
// TODO(matloob): Delete this file once Go 1.12 is released.
// This file complements golist_fallback.go by providing
// support for generating testmains.
func generateTestmain(out string, testPkg, xtestPkg *Package) (extraimports, extradeps []string, err error) {
testFuncs, err := loadTestFuncs(testPkg, xtestPkg)
if err != nil {
return nil, nil, err
}
extraimports = []string{"testing", "testing/internal/testdeps"}
if testFuncs.TestMain == nil {
extraimports = append(extraimports, "os")
}
// Transitive dependencies of ("testing", "testing/internal/testdeps").
// os is part of the transitive closure so it and its transitive dependencies are
// included regardless of whether it's imported in the template below.
extradeps = []string{
"errors",
"internal/cpu",
"unsafe",
"internal/bytealg",
"internal/race",
"runtime/internal/atomic",
"runtime/internal/sys",
"runtime",
"sync/atomic",
"sync",
"io",
"unicode",
"unicode/utf8",
"bytes",
"math",
"syscall",
"time",
"internal/poll",
"internal/syscall/unix",
"internal/testlog",
"os",
"math/bits",
"strconv",
"reflect",
"fmt",
"sort",
"strings",
"flag",
"runtime/debug",
"context",
"runtime/trace",
"testing",
"bufio",
"regexp/syntax",
"regexp",
"compress/flate",
"encoding/binary",
"hash",
"hash/crc32",
"compress/gzip",
"path/filepath",
"io/ioutil",
"text/tabwriter",
"runtime/pprof",
"testing/internal/testdeps",
}
return extraimports, extradeps, writeTestmain(out, testFuncs)
}
// The following is adapted from the cmd/go testmain generation code.
// isTestFunc tells whether fn has the type of a testing function. arg
// specifies the parameter type we look for: B, M or T.
func isTestFunc(fn *ast.FuncDecl, arg string) bool {
if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 ||
fn.Type.Params.List == nil ||
len(fn.Type.Params.List) != 1 ||
len(fn.Type.Params.List[0].Names) > 1 {
return false
}
ptr, ok := fn.Type.Params.List[0].Type.(*ast.StarExpr)
if !ok {
return false
}
// We can't easily check that the type is *testing.M
// because we don't know how testing has been imported,
// but at least check that it's *M or *something.M.
// Same applies for B and T.
if name, ok := ptr.X.(*ast.Ident); ok && name.Name == arg {
return true
}
if sel, ok := ptr.X.(*ast.SelectorExpr); ok && sel.Sel.Name == arg {
return true
}
return false
}
// isTest tells whether name looks like a test (or benchmark, according to prefix).
// It is a Test (say) if there is a character after Test that is not a lower-case letter.
// We don't want TesticularCancer.
func isTest(name, prefix string) bool {
if !strings.HasPrefix(name, prefix) {
return false
}
if len(name) == len(prefix) { // "Test" is ok
return true
}
rune, _ := utf8.DecodeRuneInString(name[len(prefix):])
return !unicode.IsLower(rune)
}
// loadTestFuncs returns the testFuncs describing the tests that will be run.
func loadTestFuncs(ptest, pxtest *Package) (*testFuncs, error) {
t := &testFuncs{
TestPackage: ptest,
XTestPackage: pxtest,
}
for _, file := range ptest.GoFiles {
if !strings.HasSuffix(file, "_test.go") {
continue
}
if err := t.load(file, "_test", &t.ImportTest, &t.NeedTest); err != nil {
return nil, err
}
}
if pxtest != nil {
for _, file := range pxtest.GoFiles {
if err := t.load(file, "_xtest", &t.ImportXtest, &t.NeedXtest); err != nil {
return nil, err
}
}
}
return t, nil
}
// writeTestmain writes the _testmain.go file for t to the file named out.
func writeTestmain(out string, t *testFuncs) error {
f, err := os.Create(out)
if err != nil {
return err
}
defer f.Close()
if err := testmainTmpl.Execute(f, t); err != nil {
return err
}
return nil
}
type testFuncs struct {
Tests []testFunc
Benchmarks []testFunc
Examples []testFunc
TestMain *testFunc
TestPackage *Package
XTestPackage *Package
ImportTest bool
NeedTest bool
ImportXtest bool
NeedXtest bool
}
// Tested returns the name of the package being tested.
func (t *testFuncs) Tested() string {
return t.TestPackage.Name
}
type testFunc struct {
Package string // imported package name (_test or _xtest)
Name string // function name
Output string // output, for examples
Unordered bool // output is allowed to be unordered.
}
func (t *testFuncs) load(filename, pkg string, doImport, seen *bool) error {
var fset = token.NewFileSet()
f, err := parser.ParseFile(fset, filename, nil, parser.ParseComments)
if err != nil {
return errors.New("failed to parse test file " + filename)
}
for _, d := range f.Decls {
n, ok := d.(*ast.FuncDecl)
if !ok {
continue
}
if n.Recv != nil {
continue
}
name := n.Name.String()
switch {
case name == "TestMain":
if isTestFunc(n, "T") {
t.Tests = append(t.Tests, testFunc{pkg, name, "", false})
*doImport, *seen = true, true
continue
}
err := checkTestFunc(fset, n, "M")
if err != nil {
return err
}
if t.TestMain != nil {
return errors.New("multiple definitions of TestMain")
}
t.TestMain = &testFunc{pkg, name, "", false}
*doImport, *seen = true, true
case isTest(name, "Test"):
err := checkTestFunc(fset, n, "T")
if err != nil {
return err
}
t.Tests = append(t.Tests, testFunc{pkg, name, "", false})
*doImport, *seen = true, true
case isTest(name, "Benchmark"):
err := checkTestFunc(fset, n, "B")
if err != nil {
return err
}
t.Benchmarks = append(t.Benchmarks, testFunc{pkg, name, "", false})
*doImport, *seen = true, true
}
}
ex := doc.Examples(f)
sort.Slice(ex, func(i, j int) bool { return ex[i].Order < ex[j].Order })
for _, e := range ex {
*doImport = true // import test file whether executed or not
if e.Output == "" && !e.EmptyOutput {
// Don't run examples with no output.
continue
}
t.Examples = append(t.Examples, testFunc{pkg, "Example" + e.Name, e.Output, e.Unordered})
*seen = true
}
return nil
}
func checkTestFunc(fset *token.FileSet, fn *ast.FuncDecl, arg string) error {
if !isTestFunc(fn, arg) {
name := fn.Name.String()
pos := fset.Position(fn.Pos())
return fmt.Errorf("%s: wrong signature for %s, must be: func %s(%s *testing.%s)", pos, name, name, strings.ToLower(arg), arg)
}
return nil
}
var testmainTmpl = template.Must(template.New("main").Parse(`
package main
import (
{{if not .TestMain}}
"os"
{{end}}
"testing"
"testing/internal/testdeps"
{{if .ImportTest}}
{{if .NeedTest}}_test{{else}}_{{end}} {{.TestPackage.PkgPath | printf "%q"}}
{{end}}
{{if .ImportXtest}}
{{if .NeedXtest}}_xtest{{else}}_{{end}} {{.XTestPackage.PkgPath | printf "%q"}}
{{end}}
)
var tests = []testing.InternalTest{
{{range .Tests}}
{"{{.Name}}", {{.Package}}.{{.Name}}},
{{end}}
}
var benchmarks = []testing.InternalBenchmark{
{{range .Benchmarks}}
{"{{.Name}}", {{.Package}}.{{.Name}}},
{{end}}
}
var examples = []testing.InternalExample{
{{range .Examples}}
{"{{.Name}}", {{.Package}}.{{.Name}}, {{.Output | printf "%q"}}, {{.Unordered}}},
{{end}}
}
func init() {
testdeps.ImportPath = {{.TestPackage.PkgPath | printf "%q"}}
}
func main() {
m := testing.MainStart(testdeps.TestDeps{}, tests, benchmarks, examples)
{{with .TestMain}}
{{.Package}}.{{.Name}}(m)
{{else}}
os.Exit(m.Run())
{{end}}
}
`))
package packages
import (
"go/parser"
"go/token"
"path/filepath"
"strconv"
"strings"
)
// processGolistOverlay provides rudimentary support for adding
// files that don't exist on disk to an overlay. The results can be
// sometimes incorrect.
// TODO(matloob): Handle unsupported cases, including the following:
// - test files
// - adding test and non-test files to test variants of packages
// - determining the correct package to add given a new import path
// - creating packages that don't exist
func processGolistOverlay(cfg *Config, response *driverResponse) (modifiedPkgs, needPkgs []string, err error) {
havePkgs := make(map[string]string) // importPath -> non-test package ID
needPkgsSet := make(map[string]bool)
modifiedPkgsSet := make(map[string]bool)
for _, pkg := range response.Packages {
// This is an approximation of import path to id. This can be
// wrong for tests, vendored packages, and a number of other cases.
havePkgs[pkg.PkgPath] = pkg.ID
}
outer:
for path, contents := range cfg.Overlay {
base := filepath.Base(path)
if strings.HasSuffix(path, "_test.go") {
// Overlays don't support adding new test files yet.
// TODO(matloob): support adding new test files.
continue
}
dir := filepath.Dir(path)
for _, pkg := range response.Packages {
var dirContains, fileExists bool
for _, f := range pkg.GoFiles {
if sameFile(filepath.Dir(f), dir) {
dirContains = true
}
if filepath.Base(f) == base {
fileExists = true
}
}
if dirContains {
if !fileExists {
pkg.GoFiles = append(pkg.GoFiles, path) // TODO(matloob): should the file just be added to GoFiles?
pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, path)
modifiedPkgsSet[pkg.ID] = true
}
imports, err := extractImports(path, contents)
if err != nil {
// Let the parser or type checker report errors later.
continue outer
}
for _, imp := range imports {
_, found := pkg.Imports[imp]
if !found {
needPkgsSet[imp] = true
// TODO(matloob): Handle cases when the following block isn't correct.
// These include imports of test variants, imports of vendored packages, etc.
id, ok := havePkgs[imp]
if !ok {
id = imp
}
pkg.Imports[imp] = &Package{ID: id}
}
}
continue outer
}
}
}
needPkgs = make([]string, 0, len(needPkgsSet))
for pkg := range needPkgsSet {
needPkgs = append(needPkgs, pkg)
}
modifiedPkgs = make([]string, 0, len(modifiedPkgsSet))
for pkg := range modifiedPkgsSet {
modifiedPkgs = append(modifiedPkgs, pkg)
}
return modifiedPkgs, needPkgs, err
}
func extractImports(filename string, contents []byte) ([]string, error) {
f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.ImportsOnly) // TODO(matloob): reuse fileset?
if err != nil {
return nil, err
}
var res []string
for _, imp := range f.Imports {
quotedPath := imp.Path.Value
path, err := strconv.Unquote(quotedPath)
if err != nil {
return nil, err
}
res = append(res, path)
}
return res, nil
}
package packages
import (
"fmt"
"os"
"sort"
)
// Visit visits all the packages in the import graph whose roots are
// pkgs, calling the optional pre function the first time each package
// is encountered (preorder), and the optional post function after a
// package's dependencies have been visited (postorder).
// The boolean result of pre(pkg) determines whether
// the imports of package pkg are visited.
func Visit(pkgs []*Package, pre func(*Package) bool, post func(*Package)) {
seen := make(map[*Package]bool)
var visit func(*Package)
visit = func(pkg *Package) {
if !seen[pkg] {
seen[pkg] = true
if pre == nil || pre(pkg) {
paths := make([]string, 0, len(pkg.Imports))
for path := range pkg.Imports {
paths = append(paths, path)
}
sort.Strings(paths) // Imports is a map, this makes visit stable
for _, path := range paths {
visit(pkg.Imports[path])
}
}
if post != nil {
post(pkg)
}
}
}
for _, pkg := range pkgs {
visit(pkg)
}
}
// PrintErrors prints to os.Stderr the accumulated errors of all
// packages in the import graph rooted at pkgs, dependencies first.
// PrintErrors returns the number of errors printed.
func PrintErrors(pkgs []*Package) int {
var n int
Visit(pkgs, nil, func(pkg *Package) {
for _, err := range pkg.Errors {
fmt.Fprintln(os.Stderr, err)
n++
}
})
return n
}
...@@ -2,6 +2,16 @@ ...@@ -2,6 +2,16 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
// Package vcs exposes functions for resolving import paths
// and using version control systems, which can be used to
// implement behavior similar to the standard "go get" command.
//
// This package is a copy of internal code in package cmd/go/internal/get,
// modified to make the identifiers exported. It's provided here
// for developers who want to write tools with similar semantics.
// It needs to be manually kept in sync with upstream when changes are
// made to cmd/go/internal/get; see https://golang.org/issue/11490.
//
package vcs package vcs
import ( import (
...@@ -10,6 +20,7 @@ import ( ...@@ -10,6 +20,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"log" "log"
"net/url"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
...@@ -344,11 +355,28 @@ func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) { ...@@ -344,11 +355,28 @@ func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) {
return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot)
} }
var vcsRet *Cmd
var rootRet string
origDir := dir origDir := dir
for len(dir) > len(srcRoot) { for len(dir) > len(srcRoot) {
for _, vcs := range vcsList { for _, vcs := range vcsList {
if _, err := os.Stat(filepath.Join(dir, "."+vcs.Cmd)); err == nil { if _, err := os.Stat(filepath.Join(dir, "."+vcs.Cmd)); err == nil {
return vcs, filepath.ToSlash(dir[len(srcRoot)+1:]), nil root := filepath.ToSlash(dir[len(srcRoot)+1:])
// Record first VCS we find, but keep looking,
// to detect mistakes like one kind of VCS inside another.
if vcsRet == nil {
vcsRet = vcs
rootRet = root
continue
}
// Allow .git inside .git, which can arise due to submodules.
if vcsRet == vcs && vcs.Cmd == "git" {
continue
}
// Otherwise, we have one VCS inside a different VCS.
return nil, "", fmt.Errorf("directory %q uses %s, but parent %q uses %s",
filepath.Join(srcRoot, rootRet), vcsRet.Cmd, filepath.Join(srcRoot, root), vcs.Cmd)
} }
} }
...@@ -361,6 +389,10 @@ func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) { ...@@ -361,6 +389,10 @@ func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) {
dir = ndir dir = ndir
} }
if vcsRet != nil {
return vcsRet, rootRet, nil
}
return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir)
} }
...@@ -535,8 +567,8 @@ func RepoRootForImportDynamic(importPath string, verbose bool) (*RepoRoot, error ...@@ -535,8 +567,8 @@ func RepoRootForImportDynamic(importPath string, verbose bool) (*RepoRoot, error
} }
} }
if !strings.Contains(metaImport.RepoRoot, "://") { if err := validateRepoRoot(metaImport.RepoRoot); err != nil {
return nil, fmt.Errorf("%s: invalid repo root %q; no scheme", urlStr, metaImport.RepoRoot) return nil, fmt.Errorf("%s: invalid repo root %q: %v", urlStr, metaImport.RepoRoot, err)
} }
rr := &RepoRoot{ rr := &RepoRoot{
VCS: ByCmd(metaImport.VCS), VCS: ByCmd(metaImport.VCS),
...@@ -549,6 +581,19 @@ func RepoRootForImportDynamic(importPath string, verbose bool) (*RepoRoot, error ...@@ -549,6 +581,19 @@ func RepoRootForImportDynamic(importPath string, verbose bool) (*RepoRoot, error
return rr, nil return rr, nil
} }
// validateRepoRoot returns an error if repoRoot does not seem to be
// a valid URL with scheme.
func validateRepoRoot(repoRoot string) error {
url, err := url.Parse(repoRoot)
if err != nil {
return err
}
if url.Scheme == "" {
return errors.New("no scheme")
}
return nil
}
// metaImport represents the parsed <meta name="go-import" // metaImport represents the parsed <meta name="go-import"
// content="prefix vcs reporoot" /> tags from HTML files. // content="prefix vcs reporoot" /> tags from HTML files.
type metaImport struct { type metaImport struct {
...@@ -558,15 +603,28 @@ type metaImport struct { ...@@ -558,15 +603,28 @@ type metaImport struct {
// errNoMatch is returned from matchGoImport when there's no applicable match. // errNoMatch is returned from matchGoImport when there's no applicable match.
var errNoMatch = errors.New("no import match") var errNoMatch = errors.New("no import match")
// pathPrefix reports whether sub is a prefix of s,
// only considering entire path components.
func pathPrefix(s, sub string) bool {
// strings.HasPrefix is necessary but not sufficient.
if !strings.HasPrefix(s, sub) {
return false
}
// The remainder after the prefix must either be empty or start with a slash.
rem := s[len(sub):]
return rem == "" || rem[0] == '/'
}
// matchGoImport returns the metaImport from imports matching importPath. // matchGoImport returns the metaImport from imports matching importPath.
// An error is returned if there are multiple matches. // An error is returned if there are multiple matches.
// errNoMatch is returned if none match. // errNoMatch is returned if none match.
func matchGoImport(imports []metaImport, importPath string) (_ metaImport, err error) { func matchGoImport(imports []metaImport, importPath string) (_ metaImport, err error) {
match := -1 match := -1
for i, im := range imports { for i, im := range imports {
if !strings.HasPrefix(importPath, im.Prefix) { if !pathPrefix(importPath, im.Prefix) {
continue continue
} }
if match != -1 { if match != -1 {
err = fmt.Errorf("multiple meta tags match import path %q", importPath) err = fmt.Errorf("multiple meta tags match import path %q", importPath)
return return
...@@ -590,15 +648,6 @@ func expand(match map[string]string, s string) string { ...@@ -590,15 +648,6 @@ func expand(match map[string]string, s string) string {
// vcsPaths lists the known vcs paths. // vcsPaths lists the known vcs paths.
var vcsPaths = []*vcsPath{ var vcsPaths = []*vcsPath{
// go.googlesource.com
{
prefix: "go.googlesource.com",
re: `^(?P<root>go\.googlesource\.com/[A-Za-z0-9_.\-]+/?)$`,
vcs: "git",
repo: "https://{root}",
check: noVCSSuffix,
},
// Github // Github
{ {
prefix: "github.com/", prefix: "github.com/",
...@@ -673,7 +722,7 @@ func bitbucketVCS(match map[string]string) error { ...@@ -673,7 +722,7 @@ func bitbucketVCS(match map[string]string) error {
var resp struct { var resp struct {
SCM string `json:"scm"` SCM string `json:"scm"`
} }
url := expand(match, "https://api.bitbucket.org/1.0/repositories/{bitname}") url := expand(match, "https://api.bitbucket.org/2.0/repositories/{bitname}?fields=scm")
data, err := httpGET(url) data, err := httpGET(url)
if err != nil { if err != nil {
return err return err
......
...@@ -3,20 +3,21 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") ...@@ -3,20 +3,21 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library( go_library(
name = "go_default_library", name = "go_default_library",
srcs = [ srcs = [
"fastwalk.go",
"fastwalk_dirent_fileno.go",
"fastwalk_dirent_ino.go",
"fastwalk_portable.go",
"fastwalk_unix.go",
"fix.go", "fix.go",
"imports.go", "imports.go",
"mod.go",
"sortimports.go", "sortimports.go",
"zstdlib.go", "zstdlib.go",
], ],
importmap = "k8s.io/kubernetes/vendor/golang.org/x/tools/imports", importmap = "k8s.io/kubernetes/vendor/golang.org/x/tools/imports",
importpath = "golang.org/x/tools/imports", importpath = "golang.org/x/tools/imports",
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
deps = ["//vendor/golang.org/x/tools/go/ast/astutil:go_default_library"], deps = [
"//vendor/golang.org/x/tools/go/ast/astutil:go_default_library",
"//vendor/golang.org/x/tools/go/packages:go_default_library",
"//vendor/golang.org/x/tools/internal/gopathwalk:go_default_library",
"//vendor/golang.org/x/tools/internal/module:go_default_library",
],
) )
filegroup( filegroup(
......
...@@ -13,11 +13,13 @@ import ( ...@@ -13,11 +13,13 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"go/ast" "go/ast"
"go/build"
"go/format" "go/format"
"go/parser" "go/parser"
"go/printer" "go/printer"
"go/token" "go/token"
"io" "io"
"io/ioutil"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
...@@ -44,9 +46,21 @@ type Options struct { ...@@ -44,9 +46,21 @@ type Options struct {
// so it is important that filename be accurate. // so it is important that filename be accurate.
// To process data ``as if'' it were in filename, pass the data as a non-nil src. // To process data ``as if'' it were in filename, pass the data as a non-nil src.
func Process(filename string, src []byte, opt *Options) ([]byte, error) { func Process(filename string, src []byte, opt *Options) ([]byte, error) {
env := &fixEnv{GOPATH: build.Default.GOPATH, GOROOT: build.Default.GOROOT}
return process(filename, src, opt, env)
}
func process(filename string, src []byte, opt *Options, env *fixEnv) ([]byte, error) {
if opt == nil { if opt == nil {
opt = &Options{Comments: true, TabIndent: true, TabWidth: 8} opt = &Options{Comments: true, TabIndent: true, TabWidth: 8}
} }
if src == nil {
b, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
src = b
}
fileSet := token.NewFileSet() fileSet := token.NewFileSet()
file, adjust, err := parse(fileSet, filename, src, opt) file, adjust, err := parse(fileSet, filename, src, opt)
...@@ -55,15 +69,13 @@ func Process(filename string, src []byte, opt *Options) ([]byte, error) { ...@@ -55,15 +69,13 @@ func Process(filename string, src []byte, opt *Options) ([]byte, error) {
} }
if !opt.FormatOnly { if !opt.FormatOnly {
_, err = fixImports(fileSet, file, filename) if err := fixImports(fileSet, file, filename, env); err != nil {
if err != nil {
return nil, err return nil, err
} }
} }
sortImports(fileSet, file) sortImports(fileSet, file)
imps := astutil.Imports(fileSet, file) imps := astutil.Imports(fileSet, file)
var spacesBefore []string // import paths we need spaces before var spacesBefore []string // import paths we need spaces before
for _, impSection := range imps { for _, impSection := range imps {
// Within each block of contiguous imports, see if any // Within each block of contiguous imports, see if any
...@@ -98,7 +110,10 @@ func Process(filename string, src []byte, opt *Options) ([]byte, error) { ...@@ -98,7 +110,10 @@ func Process(filename string, src []byte, opt *Options) ([]byte, error) {
out = adjust(src, out) out = adjust(src, out)
} }
if len(spacesBefore) > 0 { if len(spacesBefore) > 0 {
out = addImportSpaces(bytes.NewReader(out), spacesBefore) out, err = addImportSpaces(bytes.NewReader(out), spacesBefore)
if err != nil {
return nil, err
}
} }
out, err = format.Source(out) out, err = format.Source(out)
...@@ -133,11 +148,18 @@ func parse(fset *token.FileSet, filename string, src []byte, opt *Options) (*ast ...@@ -133,11 +148,18 @@ func parse(fset *token.FileSet, filename string, src []byte, opt *Options) (*ast
// If this is a declaration list, make it a source file // If this is a declaration list, make it a source file
// by inserting a package clause. // by inserting a package clause.
// Insert using a ;, not a newline, so that the line numbers // Insert using a ;, not a newline, so that parse errors are on
// in psrc match the ones in src. // the correct line.
psrc := append([]byte("package main;"), src...) const prefix = "package main;"
psrc := append([]byte(prefix), src...)
file, err = parser.ParseFile(fset, filename, psrc, parserMode) file, err = parser.ParseFile(fset, filename, psrc, parserMode)
if err == nil { if err == nil {
// Gofmt will turn the ; into a \n.
// Do that ourselves now and update the file contents,
// so that positions and line numbers are correct going forward.
psrc[len(prefix)-1] = '\n'
fset.File(file.Package).SetLinesForContent(psrc)
// If a main function exists, we will assume this is a main // If a main function exists, we will assume this is a main
// package and leave the file. // package and leave the file.
if containsMainFunc(file) { if containsMainFunc(file) {
...@@ -146,8 +168,7 @@ func parse(fset *token.FileSet, filename string, src []byte, opt *Options) (*ast ...@@ -146,8 +168,7 @@ func parse(fset *token.FileSet, filename string, src []byte, opt *Options) (*ast
adjust := func(orig, src []byte) []byte { adjust := func(orig, src []byte) []byte {
// Remove the package clause. // Remove the package clause.
// Gofmt has turned the ; into a \n. src = src[len(prefix):]
src = src[len("package main\n"):]
return matchSpace(orig, src) return matchSpace(orig, src)
} }
return file, adjust, nil return file, adjust, nil
...@@ -256,13 +277,18 @@ func matchSpace(orig []byte, src []byte) []byte { ...@@ -256,13 +277,18 @@ func matchSpace(orig []byte, src []byte) []byte {
var impLine = regexp.MustCompile(`^\s+(?:[\w\.]+\s+)?"(.+)"`) var impLine = regexp.MustCompile(`^\s+(?:[\w\.]+\s+)?"(.+)"`)
func addImportSpaces(r io.Reader, breaks []string) []byte { func addImportSpaces(r io.Reader, breaks []string) ([]byte, error) {
var out bytes.Buffer var out bytes.Buffer
sc := bufio.NewScanner(r) in := bufio.NewReader(r)
inImports := false inImports := false
done := false done := false
for sc.Scan() { for {
s := sc.Text() s, err := in.ReadString('\n')
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
if !inImports && !done && strings.HasPrefix(s, "import") { if !inImports && !done && strings.HasPrefix(s, "import") {
inImports = true inImports = true
...@@ -283,7 +309,7 @@ func addImportSpaces(r io.Reader, breaks []string) []byte { ...@@ -283,7 +309,7 @@ func addImportSpaces(r io.Reader, breaks []string) []byte {
} }
} }
fmt.Fprintln(&out, s) fmt.Fprint(&out, s)
} }
return out.Bytes() return out.Bytes(), nil
} }
...@@ -14,9 +14,9 @@ import ( ...@@ -14,9 +14,9 @@ import (
"io/ioutil" "io/ioutil"
"log" "log"
"os" "os"
"path"
"path/filepath" "path/filepath"
"regexp" "regexp"
"runtime"
"sort" "sort"
"strings" "strings"
) )
...@@ -30,11 +30,13 @@ func mustOpen(name string) io.Reader { ...@@ -30,11 +30,13 @@ func mustOpen(name string) io.Reader {
} }
func api(base string) string { func api(base string) string {
return filepath.Join(os.Getenv("GOROOT"), "api", base) return filepath.Join(runtime.GOROOT(), "api", base)
} }
var sym = regexp.MustCompile(`^pkg (\S+).*?, (?:var|func|type|const) ([A-Z]\w*)`) var sym = regexp.MustCompile(`^pkg (\S+).*?, (?:var|func|type|const) ([A-Z]\w*)`)
var unsafeSyms = map[string]bool{"Alignof": true, "ArbitraryType": true, "Offsetof": true, "Pointer": true, "Sizeof": true}
func main() { func main() {
var buf bytes.Buffer var buf bytes.Buffer
outf := func(format string, args ...interface{}) { outf := func(format string, args ...interface{}) {
...@@ -42,7 +44,7 @@ func main() { ...@@ -42,7 +44,7 @@ func main() {
} }
outf("// Code generated by mkstdlib.go. DO NOT EDIT.\n\n") outf("// Code generated by mkstdlib.go. DO NOT EDIT.\n\n")
outf("package imports\n") outf("package imports\n")
outf("var stdlib = map[string]string{\n") outf("var stdlib = map[string]map[string]bool{\n")
f := io.MultiReader( f := io.MultiReader(
mustOpen(api("go1.txt")), mustOpen(api("go1.txt")),
mustOpen(api("go1.1.txt")), mustOpen(api("go1.1.txt")),
...@@ -53,11 +55,18 @@ func main() { ...@@ -53,11 +55,18 @@ func main() {
mustOpen(api("go1.6.txt")), mustOpen(api("go1.6.txt")),
mustOpen(api("go1.7.txt")), mustOpen(api("go1.7.txt")),
mustOpen(api("go1.8.txt")), mustOpen(api("go1.8.txt")),
mustOpen(api("go1.9.txt")),
mustOpen(api("go1.10.txt")),
mustOpen(api("go1.11.txt")),
mustOpen(api("go1.12.txt")),
) )
sc := bufio.NewScanner(f) sc := bufio.NewScanner(f)
fullImport := map[string]string{} // "zip.NewReader" => "archive/zip"
ambiguous := map[string]bool{} pkgs := map[string]map[string]bool{
var keys []string "unsafe": unsafeSyms,
}
paths := []string{"unsafe"}
for sc.Scan() { for sc.Scan() {
l := sc.Text() l := sc.Text()
has := func(v string) bool { return strings.Contains(l, v) } has := func(v string) bool { return strings.Contains(l, v) }
...@@ -65,32 +74,31 @@ func main() { ...@@ -65,32 +74,31 @@ func main() {
continue continue
} }
if m := sym.FindStringSubmatch(l); m != nil { if m := sym.FindStringSubmatch(l); m != nil {
full := m[1] path, sym := m[1], m[2]
key := path.Base(full) + "." + m[2]
if exist, ok := fullImport[key]; ok { if _, ok := pkgs[path]; !ok {
if exist != full { pkgs[path] = map[string]bool{}
ambiguous[key] = true paths = append(paths, path)
}
} else {
fullImport[key] = full
keys = append(keys, key)
} }
pkgs[path][sym] = true
} }
} }
if err := sc.Err(); err != nil { if err := sc.Err(); err != nil {
log.Fatal(err) log.Fatal(err)
} }
sort.Strings(keys) sort.Strings(paths)
for _, key := range keys { for _, path := range paths {
if ambiguous[key] { outf("\t%q: map[string]bool{\n", path)
outf("\t// %q is ambiguous\n", key) pkg := pkgs[path]
} else { var syms []string
outf("\t%q: %q,\n", key, fullImport[key]) for sym := range pkg {
syms = append(syms, sym)
} }
} sort.Strings(syms)
outf("\n") for _, sym := range syms {
for _, sym := range [...]string{"Alignof", "ArbitraryType", "Offsetof", "Pointer", "Sizeof"} { outf("\t\t%q: true,\n", sym)
outf("\t%q: %q,\n", "unsafe."+sym, "unsafe") }
outf("},\n")
} }
outf("}\n") outf("}\n")
fmtbuf, err := format.Source(buf.Bytes()) fmtbuf, err := format.Source(buf.Bytes())
......
package imports
import (
"bytes"
"encoding/json"
"io/ioutil"
"log"
"os"
"path"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/tools/internal/gopathwalk"
"golang.org/x/tools/internal/module"
)
// moduleResolver implements resolver for modules using the go command as little
// as feasible.
type moduleResolver struct {
env *fixEnv
main *moduleJSON
modsByModPath []*moduleJSON // All modules, ordered by # of path components in module Path...
modsByDir []*moduleJSON // ...or Dir.
}
type moduleJSON struct {
Path string // module path
Version string // module version
Versions []string // available module versions (with -versions)
Replace *moduleJSON // replaced by this module
Time *time.Time // time version was created
Update *moduleJSON // available update, if any (with -u)
Main bool // is this the main module?
Indirect bool // is this module only an indirect dependency of main module?
Dir string // directory holding files for this module, if any
GoMod string // path to go.mod file for this module, if any
Error *moduleErrorJSON // error loading module
}
type moduleErrorJSON struct {
Err string // the error itself
}
func (r *moduleResolver) init() error {
if r.main != nil {
return nil
}
stdout, err := r.env.invokeGo("list", "-m", "-json", "...")
if err != nil {
return err
}
for dec := json.NewDecoder(stdout); dec.More(); {
mod := &moduleJSON{}
if err := dec.Decode(mod); err != nil {
return err
}
if mod.Dir == "" {
if Debug {
log.Printf("module %v has not been downloaded and will be ignored", mod.Path)
}
// Can't do anything with a module that's not downloaded.
continue
}
r.modsByModPath = append(r.modsByModPath, mod)
r.modsByDir = append(r.modsByDir, mod)
if mod.Main {
r.main = mod
}
}
sort.Slice(r.modsByModPath, func(i, j int) bool {
count := func(x int) int {
return strings.Count(r.modsByModPath[x].Path, "/")
}
return count(j) < count(i) // descending order
})
sort.Slice(r.modsByDir, func(i, j int) bool {
count := func(x int) int {
return strings.Count(r.modsByDir[x].Dir, "/")
}
return count(j) < count(i) // descending order
})
return nil
}
// findPackage returns the module and directory that contains the package at
// the given import path, or returns nil, "" if no module is in scope.
func (r *moduleResolver) findPackage(importPath string) (*moduleJSON, string) {
for _, m := range r.modsByModPath {
if !strings.HasPrefix(importPath, m.Path) {
continue
}
pathInModule := importPath[len(m.Path):]
pkgDir := filepath.Join(m.Dir, pathInModule)
if dirIsNestedModule(pkgDir, m) {
continue
}
pkgFiles, err := ioutil.ReadDir(pkgDir)
if err != nil {
continue
}
// A module only contains a package if it has buildable go
// files in that directory. If not, it could be provided by an
// outer module. See #29736.
for _, fi := range pkgFiles {
if ok, _ := r.env.buildContext().MatchFile(pkgDir, fi.Name()); ok {
return m, pkgDir
}
}
}
return nil, ""
}
// findModuleByDir returns the module that contains dir, or nil if no such
// module is in scope.
func (r *moduleResolver) findModuleByDir(dir string) *moduleJSON {
// This is quite tricky and may not be correct. dir could be:
// - a package in the main module.
// - a replace target underneath the main module's directory.
// - a nested module in the above.
// - a replace target somewhere totally random.
// - a nested module in the above.
// - in the mod cache.
// - in /vendor/ in -mod=vendor mode.
// - nested module? Dunno.
// Rumor has it that replace targets cannot contain other replace targets.
for _, m := range r.modsByDir {
if !strings.HasPrefix(dir, m.Dir) {
continue
}
if dirIsNestedModule(dir, m) {
continue
}
return m
}
return nil
}
// dirIsNestedModule reports if dir is contained in a nested module underneath
// mod, not actually in mod.
func dirIsNestedModule(dir string, mod *moduleJSON) bool {
if !strings.HasPrefix(dir, mod.Dir) {
return false
}
mf := findModFile(dir)
if mf == "" {
return false
}
return filepath.Dir(mf) != mod.Dir
}
func findModFile(dir string) string {
for {
f := filepath.Join(dir, "go.mod")
info, err := os.Stat(f)
if err == nil && !info.IsDir() {
return f
}
d := filepath.Dir(dir)
if len(d) >= len(dir) {
return "" // reached top of file system, no go.mod
}
dir = d
}
}
func (r *moduleResolver) loadPackageNames(importPaths []string, srcDir string) (map[string]string, error) {
if err := r.init(); err != nil {
return nil, err
}
names := map[string]string{}
for _, path := range importPaths {
_, packageDir := r.findPackage(path)
if packageDir == "" {
continue
}
name, err := packageDirToName(packageDir)
if err != nil {
continue
}
names[path] = name
}
return names, nil
}
func (r *moduleResolver) scan(_ references) ([]*pkg, error) {
if err := r.init(); err != nil {
return nil, err
}
// Walk GOROOT, GOPATH/pkg/mod, and the main module.
roots := []gopathwalk.Root{
{filepath.Join(r.env.GOROOT, "/src"), gopathwalk.RootGOROOT},
{r.main.Dir, gopathwalk.RootCurrentModule},
}
for _, p := range filepath.SplitList(r.env.GOPATH) {
roots = append(roots, gopathwalk.Root{filepath.Join(p, "/pkg/mod"), gopathwalk.RootModuleCache})
}
// Walk replace targets, just in case they're not in any of the above.
for _, mod := range r.modsByModPath {
if mod.Replace != nil {
roots = append(roots, gopathwalk.Root{mod.Dir, gopathwalk.RootOther})
}
}
var result []*pkg
dupCheck := make(map[string]bool)
var mu sync.Mutex
gopathwalk.Walk(roots, func(root gopathwalk.Root, dir string) {
mu.Lock()
defer mu.Unlock()
if _, dup := dupCheck[dir]; dup {
return
}
dupCheck[dir] = true
subdir := ""
if dir != root.Path {
subdir = dir[len(root.Path)+len("/"):]
}
importPath := filepath.ToSlash(subdir)
if strings.HasPrefix(importPath, "vendor/") {
// Ignore vendor dirs. If -mod=vendor is on, then things
// should mostly just work, but when it's not vendor/
// is a mess. There's no easy way to tell if it's on.
// We can still find things in the mod cache and
// map them into /vendor when -mod=vendor is on.
return
}
switch root.Type {
case gopathwalk.RootCurrentModule:
importPath = path.Join(r.main.Path, filepath.ToSlash(subdir))
case gopathwalk.RootModuleCache:
matches := modCacheRegexp.FindStringSubmatch(subdir)
modPath, err := module.DecodePath(filepath.ToSlash(matches[1]))
if err != nil {
if Debug {
log.Printf("decoding module cache path %q: %v", subdir, err)
}
return
}
importPath = path.Join(modPath, filepath.ToSlash(matches[3]))
case gopathwalk.RootGOROOT:
importPath = subdir
}
// Check if the directory is underneath a module that's in scope.
if mod := r.findModuleByDir(dir); mod != nil {
// It is. If dir is the target of a replace directive,
// our guessed import path is wrong. Use the real one.
if mod.Dir == dir {
importPath = mod.Path
} else {
dirInMod := dir[len(mod.Dir)+len("/"):]
importPath = path.Join(mod.Path, filepath.ToSlash(dirInMod))
}
} else {
// The package is in an unknown module. Check that it's
// not obviously impossible to import.
var modFile string
switch root.Type {
case gopathwalk.RootModuleCache:
matches := modCacheRegexp.FindStringSubmatch(subdir)
modFile = filepath.Join(matches[1], "@", matches[2], "go.mod")
default:
modFile = findModFile(dir)
}
modBytes, err := ioutil.ReadFile(modFile)
if err == nil && !strings.HasPrefix(importPath, modulePath(modBytes)) {
// The module's declared path does not match
// its expected path. It probably needs a
// replace directive we don't have.
return
}
}
// We may have discovered a package that has a different version
// in scope already. Canonicalize to that one if possible.
if _, canonicalDir := r.findPackage(importPath); canonicalDir != "" {
dir = canonicalDir
}
result = append(result, &pkg{
importPathShort: VendorlessPath(importPath),
dir: dir,
})
}, gopathwalk.Options{Debug: Debug, ModulesEnabled: true})
return result, nil
}
// modCacheRegexp splits a path in a module cache into module, module version, and package.
var modCacheRegexp = regexp.MustCompile(`(.*)@([^/\\]*)(.*)`)
var (
slashSlash = []byte("//")
moduleStr = []byte("module")
)
// modulePath returns the module path from the gomod file text.
// If it cannot find a module path, it returns an empty string.
// It is tolerant of unrelated problems in the go.mod file.
//
// Copied from cmd/go/internal/modfile.
func modulePath(mod []byte) string {
for len(mod) > 0 {
line := mod
mod = nil
if i := bytes.IndexByte(line, '\n'); i >= 0 {
line, mod = line[:i], line[i+1:]
}
if i := bytes.Index(line, slashSlash); i >= 0 {
line = line[:i]
}
line = bytes.TrimSpace(line)
if !bytes.HasPrefix(line, moduleStr) {
continue
}
line = line[len(moduleStr):]
n := len(line)
line = bytes.TrimSpace(line)
if len(line) == n || len(line) == 0 {
continue
}
if line[0] == '"' || line[0] == '`' {
p, err := strconv.Unquote(string(line))
if err != nil {
return "" // malformed quoted string or multiline module path
}
return p
}
return string(line)
}
return "" // missing module path
}
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment