Unverified Commit 30c7723c authored by Brian Downs's avatar Brian Downs Committed by GitHub

[Engine-1.21] All bootstrap backport (#4451)

Add ability to reconcile bootstrap data between datastore and disk (#3398)
parent 67d079ec
...@@ -70,6 +70,8 @@ replace ( ...@@ -70,6 +70,8 @@ replace (
require ( require (
github.com/Microsoft/hcsshim v0.8.20 github.com/Microsoft/hcsshim v0.8.20
github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40 // indirect github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40 // indirect
github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 // indirect
github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 // indirect
github.com/containerd/cgroups v1.0.1 github.com/containerd/cgroups v1.0.1
github.com/containerd/containerd v1.5.1 github.com/containerd/containerd v1.5.1
github.com/containerd/cri v1.11.1-0.20200820101445-b0cc07999aa5 github.com/containerd/cri v1.11.1-0.20200820101445-b0cc07999aa5
...@@ -81,13 +83,19 @@ require ( ...@@ -81,13 +83,19 @@ require (
github.com/docker/docker v20.10.5+incompatible github.com/docker/docker v20.10.5+incompatible
github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83 github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83
github.com/flannel-io/flannel v0.14.0 github.com/flannel-io/flannel v0.14.0
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
github.com/go-bindata/go-bindata v3.1.2+incompatible github.com/go-bindata/go-bindata v3.1.2+incompatible
github.com/go-sql-driver/mysql v1.4.1 github.com/go-sql-driver/mysql v1.4.1
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golangplus/testing v1.0.0 // indirect github.com/golangplus/testing v1.0.0 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/cadvisor v0.39.0 github.com/google/cadvisor v0.39.0
github.com/google/uuid v1.2.0 github.com/google/uuid v1.2.0
github.com/gorilla/mux v1.8.0 github.com/gorilla/mux v1.8.0
github.com/gorilla/websocket v1.4.2 github.com/gorilla/websocket v1.4.2
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/k3s-io/helm-controller v0.11.7 github.com/k3s-io/helm-controller v0.11.7
github.com/k3s-io/kine v0.6.2 github.com/k3s-io/kine v0.6.2
github.com/klauspost/compress v1.12.2 github.com/klauspost/compress v1.12.2
...@@ -104,6 +112,7 @@ require ( ...@@ -104,6 +112,7 @@ require (
github.com/opencontainers/selinux v1.8.2 github.com/opencontainers/selinux v1.8.2
github.com/pierrec/lz4 v2.6.0+incompatible github.com/pierrec/lz4 v2.6.0+incompatible
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.11.0 // indirect
github.com/rancher/dynamiclistener v0.2.6 github.com/rancher/dynamiclistener v0.2.6
github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08 github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08
github.com/rancher/remotedialer v0.2.0 github.com/rancher/remotedialer v0.2.0
...@@ -112,15 +121,21 @@ require ( ...@@ -112,15 +121,21 @@ require (
github.com/robfig/cron/v3 v3.0.1 github.com/robfig/cron/v3 v3.0.1
github.com/rootless-containers/rootlesskit v0.14.0 github.com/rootless-containers/rootlesskit v0.14.0
github.com/sirupsen/logrus v1.8.1 github.com/sirupsen/logrus v1.8.1
github.com/soheilhy/cmux v0.1.5 // indirect
github.com/spf13/cobra v1.1.3 // indirect
github.com/spf13/pflag v1.0.5 github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.7.0 github.com/stretchr/testify v1.7.0
github.com/tchap/go-patricia v2.3.0+incompatible // indirect github.com/tchap/go-patricia v2.3.0+incompatible // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
github.com/urfave/cli v1.22.2 github.com/urfave/cli v1.22.2
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd v0.5.0-alpha.5.0.20201208200253-50621aee4aea go.etcd.io/etcd v0.5.0-alpha.5.0.20201208200253-50621aee4aea
go.uber.org/zap v1.17.0 // indirect
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40
google.golang.org/grpc v1.37.0 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
google.golang.org/grpc v1.38.0
gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v2 v2.4.0
inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252 inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252
k8s.io/api v0.21.6 k8s.io/api v0.21.6
......
...@@ -7,52 +7,78 @@ import ( ...@@ -7,52 +7,78 @@ import (
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
"time"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/k3s/pkg/daemons/config"
"github.com/sirupsen/logrus"
) )
func Handler(bootstrap *config.ControlRuntimeBootstrap) http.Handler { func Handler(bootstrap *config.ControlRuntimeBootstrap) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
rw.Header().Set("Content-Type", "application/json") rw.Header().Set("Content-Type", "application/json")
Write(rw, bootstrap) ReadFromDisk(rw, bootstrap)
}) })
} }
func Write(w io.Writer, bootstrap *config.ControlRuntimeBootstrap) error { // ReadFromDisk reads the bootstrap data from the files on disk and
paths, err := objToMap(bootstrap) // writes their content in JSON form to the given io.Writer.
func ReadFromDisk(w io.Writer, bootstrap *config.ControlRuntimeBootstrap) error {
paths, err := ObjToMap(bootstrap)
if err != nil { if err != nil {
return nil return nil
} }
dataMap := map[string][]byte{} dataMap := make(map[string]File)
for pathKey, path := range paths { for pathKey, path := range paths {
if path == "" { if path == "" {
continue continue
} }
data, err := ioutil.ReadFile(path) data, err := ioutil.ReadFile(path)
if err != nil { if err != nil {
return errors.Wrapf(err, "failed to read %s", path) logrus.Warnf("failed to read %s", path)
continue
} }
dataMap[pathKey] = data info, err := os.Stat(path)
if err != nil {
return err
}
dataMap[pathKey] = File{
Timestamp: info.ModTime(),
Content: data,
}
} }
return json.NewEncoder(w).Encode(dataMap) return json.NewEncoder(w).Encode(dataMap)
} }
func Read(r io.Reader, bootstrap *config.ControlRuntimeBootstrap) error { // File is a representation of a certificate
paths, err := objToMap(bootstrap) // or key file within the bootstrap context that contains
// the contents of the file as well as a timestamp from
// when the file was last modified.
type File struct {
Timestamp time.Time
Content []byte
}
type PathsDataformat map[string]File
// WriteToDiskFromStorage writes the contents of the given reader to the paths
// derived from within the ControlRuntimeBootstrap.
func WriteToDiskFromStorage(r io.Reader, bootstrap *config.ControlRuntimeBootstrap) error {
paths, err := ObjToMap(bootstrap)
if err != nil { if err != nil {
return err return err
} }
files := map[string][]byte{} files := make(PathsDataformat)
if err := json.NewDecoder(r).Decode(&files); err != nil { if err := json.NewDecoder(r).Decode(&files); err != nil {
return err return err
} }
for pathKey, data := range files { for pathKey, bsf := range files {
path, ok := paths[pathKey] path, ok := paths[pathKey]
if !ok { if !ok {
continue continue
...@@ -61,8 +87,7 @@ func Read(r io.Reader, bootstrap *config.ControlRuntimeBootstrap) error { ...@@ -61,8 +87,7 @@ func Read(r io.Reader, bootstrap *config.ControlRuntimeBootstrap) error {
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return errors.Wrapf(err, "failed to mkdir %s", filepath.Dir(path)) return errors.Wrapf(err, "failed to mkdir %s", filepath.Dir(path))
} }
if err := ioutil.WriteFile(path, bsf.Content, 0600); err != nil {
if err := ioutil.WriteFile(path, data, 0600); err != nil {
return errors.Wrapf(err, "failed to write to %s", path) return errors.Wrapf(err, "failed to write to %s", path)
} }
} }
...@@ -70,7 +95,7 @@ func Read(r io.Reader, bootstrap *config.ControlRuntimeBootstrap) error { ...@@ -70,7 +95,7 @@ func Read(r io.Reader, bootstrap *config.ControlRuntimeBootstrap) error {
return nil return nil
} }
func objToMap(obj interface{}) (map[string]string, error) { func ObjToMap(obj interface{}) (map[string]string, error) {
bytes, err := json.Marshal(obj) bytes, err := json.Marshal(obj)
if err != nil { if err != nil {
return nil, err return nil, err
......
package bootstrap
import (
"testing"
"github.com/rancher/k3s/pkg/daemons/config"
)
func TestObjToMap(t *testing.T) {
type args struct {
obj interface{}
}
tests := []struct {
name string
args args
want map[string]string
wantErr bool
}{
{
name: "Minimal Valid",
args: args{
obj: &config.ControlRuntimeBootstrap{
ServerCA: "/var/lib/rancher/k3s/server/tls/server-ca.crt",
ServerCAKey: "/var/lib/rancher/k3s/server/tls/server-ca.key",
},
},
wantErr: false,
},
{
name: "Minimal Invalid",
args: args{
obj: 1,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := ObjToMap(tt.args.obj)
if (err != nil) != tt.wantErr {
t.Errorf("ObjToMap() error = %v, wantErr %v", err, tt.wantErr)
return
}
})
}
}
...@@ -95,7 +95,7 @@ func run(app *cli.Context, cfg *cmds.Server) error { ...@@ -95,7 +95,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
cluster := cluster.New(&serverConfig.ControlConfig) cluster := cluster.New(&serverConfig.ControlConfig)
if err := cluster.Bootstrap(ctx); err != nil { if err := cluster.Bootstrap(ctx, true); err != nil {
return err return err
} }
......
...@@ -176,6 +176,17 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont ...@@ -176,6 +176,17 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
// delete local loadbalancers state for apiserver and supervisor servers // delete local loadbalancers state for apiserver and supervisor servers
loadbalancer.ResetLoadBalancer(filepath.Join(cfg.DataDir, "agent"), loadbalancer.SupervisorServiceName) loadbalancer.ResetLoadBalancer(filepath.Join(cfg.DataDir, "agent"), loadbalancer.SupervisorServiceName)
loadbalancer.ResetLoadBalancer(filepath.Join(cfg.DataDir, "agent"), loadbalancer.APIServerServiceName) loadbalancer.ResetLoadBalancer(filepath.Join(cfg.DataDir, "agent"), loadbalancer.APIServerServiceName)
// at this point we're doing a restore. Check to see if we've
// passed in a token and if not, check if the token file exists.
// If it doesn't, return an error indicating the token is necessary.
if cfg.Token == "" {
if _, err := os.Stat(filepath.Join(cfg.DataDir, "server/token")); err != nil {
if os.IsNotExist(err) {
return errors.New("")
}
}
}
} }
serverConfig.ControlConfig.ClusterReset = cfg.ClusterReset serverConfig.ControlConfig.ClusterReset = cfg.ClusterReset
......
...@@ -9,7 +9,7 @@ import ( ...@@ -9,7 +9,7 @@ import (
) )
// WriteClientKubeConfig generates a kubeconfig at destFile that can be used to connect to a server at url with the given certs and keys // WriteClientKubeConfig generates a kubeconfig at destFile that can be used to connect to a server at url with the given certs and keys
func WriteClientKubeConfig(destFile string, url string, serverCAFile string, clientCertFile string, clientKeyFile string) error { func WriteClientKubeConfig(destFile, url, serverCAFile, clientCertFile, clientKeyFile string) error {
serverCA, err := ioutil.ReadFile(serverCAFile) serverCA, err := ioutil.ReadFile(serverCAFile)
if err != nil { if err != nil {
return errors.Wrapf(err, "failed to read %s", serverCAFile) return errors.Wrapf(err, "failed to read %s", serverCAFile)
......
...@@ -16,9 +16,15 @@ import ( ...@@ -16,9 +16,15 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
var ( const (
tokenPrefix = "K10"
tokenFormat = "%s%s::%s:%s"
caHashLength = sha256.Size * 2
defaultClientTimeout = 10 * time.Second defaultClientTimeout = 10 * time.Second
)
var (
defaultClient = &http.Client{ defaultClient = &http.Client{
Timeout: defaultClientTimeout, Timeout: defaultClientTimeout,
} }
...@@ -32,12 +38,6 @@ var ( ...@@ -32,12 +38,6 @@ var (
} }
) )
const (
tokenPrefix = "K10"
tokenFormat = "%s%s::%s:%s"
caHashLength = sha256.Size * 2
)
type OverrideURLCallback func(config []byte) (*url.URL, error) type OverrideURLCallback func(config []byte) (*url.URL, error)
type Info struct { type Info struct {
...@@ -49,8 +49,8 @@ type Info struct { ...@@ -49,8 +49,8 @@ type Info struct {
} }
// String returns the token data, templated according to the token format // String returns the token data, templated according to the token format
func (info *Info) String() string { func (i *Info) String() string {
return fmt.Sprintf(tokenFormat, tokenPrefix, hashCA(info.CACerts), info.Username, info.Password) return fmt.Sprintf(tokenFormat, tokenPrefix, hashCA(i.CACerts), i.Username, i.Password)
} }
// ParseAndValidateToken parses a token, downloads and validates the server's CA bundle, // ParseAndValidateToken parses a token, downloads and validates the server's CA bundle,
...@@ -70,7 +70,7 @@ func ParseAndValidateToken(server string, token string) (*Info, error) { ...@@ -70,7 +70,7 @@ func ParseAndValidateToken(server string, token string) (*Info, error) {
// ParseAndValidateToken parses a token with user override, downloads and // ParseAndValidateToken parses a token with user override, downloads and
// validates the server's CA bundle, and validates it according to the caHash from the token if set. // validates the server's CA bundle, and validates it according to the caHash from the token if set.
func ParseAndValidateTokenForUser(server string, token string, username string) (*Info, error) { func ParseAndValidateTokenForUser(server, token, username string) (*Info, error) {
info, err := parseToken(token) info, err := parseToken(token)
if err != nil { if err != nil {
return nil, err return nil, err
...@@ -86,11 +86,11 @@ func ParseAndValidateTokenForUser(server string, token string, username string) ...@@ -86,11 +86,11 @@ func ParseAndValidateTokenForUser(server string, token string, username string)
} }
// setAndValidateServer updates the remote server's cert info, and validates it against the provided hash // setAndValidateServer updates the remote server's cert info, and validates it against the provided hash
func (info *Info) setAndValidateServer(server string) error { func (i *Info) setAndValidateServer(server string) error {
if err := info.setServer(server); err != nil { if err := i.setServer(server); err != nil {
return err return err
} }
return info.validateCAHash() return i.validateCAHash()
} }
// validateCACerts returns a boolean indicating whether or not a CA bundle matches the provided hash, // validateCACerts returns a boolean indicating whether or not a CA bundle matches the provided hash,
...@@ -118,7 +118,7 @@ func ParseUsernamePassword(token string) (string, string, bool) { ...@@ -118,7 +118,7 @@ func ParseUsernamePassword(token string) (string, string, bool) {
// parseToken parses a token into an Info struct // parseToken parses a token into an Info struct
func parseToken(token string) (*Info, error) { func parseToken(token string) (*Info, error) {
var info = &Info{} var info Info
if len(token) == 0 { if len(token) == 0 {
return nil, errors.New("token must not be empty") return nil, errors.New("token must not be empty")
...@@ -150,7 +150,7 @@ func parseToken(token string) (*Info, error) { ...@@ -150,7 +150,7 @@ func parseToken(token string) (*Info, error) {
info.Username = parts[0] info.Username = parts[0]
info.Password = parts[1] info.Password = parts[1]
return info, nil return &info, nil
} }
// GetHTTPClient returns a http client that validates TLS server certificates using the provided CA bundle. // GetHTTPClient returns a http client that validates TLS server certificates using the provided CA bundle.
...@@ -177,25 +177,25 @@ func GetHTTPClient(cacerts []byte) *http.Client { ...@@ -177,25 +177,25 @@ func GetHTTPClient(cacerts []byte) *http.Client {
} }
// Get makes a request to a subpath of info's BaseURL // Get makes a request to a subpath of info's BaseURL
func (info *Info) Get(path string) ([]byte, error) { func (i *Info) Get(path string) ([]byte, error) {
u, err := url.Parse(info.BaseURL) u, err := url.Parse(i.BaseURL)
if err != nil { if err != nil {
return nil, err return nil, err
} }
u.Path = path u.Path = path
return get(u.String(), GetHTTPClient(info.CACerts), info.Username, info.Password) return get(u.String(), GetHTTPClient(i.CACerts), i.Username, i.Password)
} }
// setServer sets the BaseURL and CACerts fields of the Info by connecting to the server // setServer sets the BaseURL and CACerts fields of the Info by connecting to the server
// and storing the CA bundle. // and storing the CA bundle.
func (info *Info) setServer(server string) error { func (i *Info) setServer(server string) error {
url, err := url.Parse(server) url, err := url.Parse(server)
if err != nil { if err != nil {
return errors.Wrapf(err, "Invalid server url, failed to parse: %s", server) return errors.Wrapf(err, "Invalid server url, failed to parse: %s", server)
} }
if url.Scheme != "https" { if url.Scheme != "https" {
return fmt.Errorf("only https:// URLs are supported, invalid scheme: %s", server) return errors.New("only https:// URLs are supported, invalid scheme: " + server)
} }
for strings.HasSuffix(url.Path, "/") { for strings.HasSuffix(url.Path, "/") {
...@@ -207,25 +207,25 @@ func (info *Info) setServer(server string) error { ...@@ -207,25 +207,25 @@ func (info *Info) setServer(server string) error {
return err return err
} }
info.BaseURL = url.String() i.BaseURL = url.String()
info.CACerts = cacerts i.CACerts = cacerts
return nil return nil
} }
// ValidateCAHash validates that info's caHash matches the CACerts hash. // ValidateCAHash validates that info's caHash matches the CACerts hash.
func (info *Info) validateCAHash() error { func (i *Info) validateCAHash() error {
if len(info.caHash) > 0 && len(info.CACerts) == 0 { if len(i.caHash) > 0 && len(i.CACerts) == 0 {
// Warn if the user provided a CA hash but we're not going to validate because it's already trusted // Warn if the user provided a CA hash but we're not going to validate because it's already trusted
logrus.Warn("Cluster CA certificate is trusted by the host CA bundle. " + logrus.Warn("Cluster CA certificate is trusted by the host CA bundle. " +
"Token CA hash will not be validated.") "Token CA hash will not be validated.")
} else if len(info.caHash) == 0 && len(info.CACerts) > 0 { } else if len(i.caHash) == 0 && len(i.CACerts) > 0 {
// Warn if the CA is self-signed but the user didn't provide a hash to validate it against // Warn if the CA is self-signed but the user didn't provide a hash to validate it against
logrus.Warn("Cluster CA certificate is not trusted by the host CA bundle, but the token does not include a CA hash. " + logrus.Warn("Cluster CA certificate is not trusted by the host CA bundle, but the token does not include a CA hash. " +
"Use the full token from the server's node-token file to enable Cluster CA validation.") "Use the full token from the server's node-token file to enable Cluster CA validation.")
} else if len(info.CACerts) > 0 && len(info.caHash) > 0 { } else if len(i.CACerts) > 0 && len(i.caHash) > 0 {
// only verify CA hash if the server cert is not trusted by the OS CA bundle // only verify CA hash if the server cert is not trusted by the OS CA bundle
if ok, serverHash := validateCACerts(info.CACerts, info.caHash); !ok { if ok, serverHash := validateCACerts(i.CACerts, i.caHash); !ok {
return fmt.Errorf("token CA hash does not match the Cluster CA certificate hash: %s != %s", info.caHash, serverHash) return fmt.Errorf("token CA hash does not match the Cluster CA certificate hash: %s != %s", i.caHash, serverHash)
} }
} }
return nil return nil
...@@ -288,18 +288,18 @@ func get(u string, client *http.Client, username, password string) ([]byte, erro ...@@ -288,18 +288,18 @@ func get(u string, client *http.Client, username, password string) ([]byte, erro
return ioutil.ReadAll(resp.Body) return ioutil.ReadAll(resp.Body)
} }
func FormatToken(token string, certFile string) (string, error) { func FormatToken(token, certFile string) (string, error) {
if len(token) == 0 { if len(token) == 0 {
return token, nil return token, nil
} }
certHash := "" certHash := ""
if len(certFile) > 0 { if len(certFile) > 0 {
bytes, err := ioutil.ReadFile(certFile) b, err := ioutil.ReadFile(certFile)
if err != nil { if err != nil {
return "", nil return "", nil
} }
digest := sha256.Sum256(bytes) digest := sha256.Sum256(b)
certHash = tokenPrefix + hex.EncodeToString(digest[:]) + "::" certHash = tokenPrefix + hex.EncodeToString(digest[:]) + "::"
} }
return certHash + token, nil return certHash + token, nil
......
...@@ -320,7 +320,7 @@ func newTLSServer(t *testing.T, username, password string, sendWrongCA bool) *ht ...@@ -320,7 +320,7 @@ func newTLSServer(t *testing.T, username, password string, sendWrongCA bool) *ht
} }
bootstrapData := &config.ControlRuntimeBootstrap{} bootstrapData := &config.ControlRuntimeBootstrap{}
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
if err := bootstrap.Write(w, bootstrapData); err != nil { if err := bootstrap.ReadFromDisk(w, bootstrapData); err != nil {
t.Errorf("failed to write bootstrap: %v", err) t.Errorf("failed to write bootstrap: %v", err)
} }
return return
......
package cluster
import (
"bytes"
"context"
"os"
"path/filepath"
"testing"
"time"
"github.com/k3s-io/kine/pkg/endpoint"
"github.com/rancher/k3s/pkg/bootstrap"
"github.com/rancher/k3s/pkg/clientaccess"
"github.com/rancher/k3s/pkg/cluster/managed"
"github.com/rancher/k3s/pkg/daemons/config"
)
func Test_isDirEmpty(t *testing.T) {
const tmpDir = "test_dir"
type args struct {
name string
}
tests := []struct {
name string
args args
setup func() error
teardown func() error
want bool
wantErr bool
}{
{
name: "is empty",
args: args{
name: tmpDir,
},
setup: func() error {
return os.Mkdir(tmpDir, 0700)
},
teardown: func() error {
return os.RemoveAll(tmpDir)
},
want: true,
wantErr: false,
},
{
name: "is not empty",
args: args{
name: tmpDir,
},
setup: func() error {
os.Mkdir(tmpDir, 0700)
_, _ = os.Create(filepath.Join(filepath.Dir(tmpDir), tmpDir, "test_file"))
return nil
},
teardown: func() error {
return os.RemoveAll(tmpDir)
},
want: false,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer tt.teardown()
if err := tt.setup(); err != nil {
t.Errorf("Setup for isDirEmpty() failed = %v", err)
return
}
got, err := isDirEmpty(tt.args.name)
if (err != nil) != tt.wantErr {
t.Errorf("isDirEmpty() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("isDirEmpty() = %+v\nWant = %+v", got, tt.want)
}
})
}
}
func TestCluster_certDirsExist(t *testing.T) {
const testDataDir = "/tmp/k3s/"
testTLSDir := filepath.Join(testDataDir, "server", "tls")
testTLSEtcdDir := filepath.Join(testDataDir, "server", "tls", "etcd")
type fields struct {
clientAccessInfo *clientaccess.Info
config *config.Control
runtime *config.ControlRuntime
managedDB managed.Driver
etcdConfig endpoint.ETCDConfig
shouldBootstrap bool
storageStarted bool
saveBootstrap bool
}
tests := []struct {
name string
fields fields
setup func() error
teardown func() error
wantErr bool
}{
{
name: "exists",
fields: fields{
config: &config.Control{
DataDir: filepath.Join(testDataDir, "server"),
},
},
setup: func() error {
os.MkdirAll(testTLSEtcdDir, 0700)
_, _ = os.Create(filepath.Join(testTLSDir, "test_file"))
_, _ = os.Create(filepath.Join(testTLSEtcdDir, "test_file"))
return nil
},
teardown: func() error {
return os.RemoveAll(testDataDir)
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &Cluster{
clientAccessInfo: tt.fields.clientAccessInfo,
config: tt.fields.config,
runtime: tt.fields.runtime,
managedDB: tt.fields.managedDB,
etcdConfig: tt.fields.etcdConfig,
storageStarted: tt.fields.storageStarted,
saveBootstrap: tt.fields.saveBootstrap,
}
defer tt.teardown()
if err := tt.setup(); err != nil {
t.Errorf("Setup for Cluster.certDirsExist() failed = %v", err)
return
}
if err := c.certDirsExist(); (err != nil) != tt.wantErr {
t.Errorf("Cluster.certDirsExist() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestCluster_migrateBootstrapData(t *testing.T) {
type fields struct {
clientAccessInfo *clientaccess.Info
config *config.Control
runtime *config.ControlRuntime
managedDB managed.Driver
etcdConfig endpoint.ETCDConfig
joining bool
storageStarted bool
saveBootstrap bool
shouldBootstrap bool
}
type args struct {
ctx context.Context
data *bytes.Buffer
files bootstrap.PathsDataformat
}
tests := []struct {
name string
args args
setup func() error // Optional, delete if unused
teardown func() error // Optional, delete if unused
wantErr bool
}{
{
name: "Success",
args: args{
ctx: context.Background(),
data: bytes.NewBuffer([]byte(`{"ServerCA": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlJQ3ZEQ0NBYVFDQVFBd2R6RUxNQWtHQTFVRUJoTUNWVk14RFRBTEJnTlZCQWdNQkZWMFlXZ3hEekFOQmdOVgpCQWNNQmt4cGJtUnZiakVXTUJRR0ExVUVDZ3dOUkdsbmFVTmxjblFnU1c1akxqRVJNQThHQTFVRUN3d0lSR2xuCmFVTmxjblF4SFRBYkJnTlZCQU1NRkdWNFlXMXdiR1V1WkdsbmFXTmxjblF1WTI5dE1JSUJJakFOQmdrcWhraUcKOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQTgrVG83ZCsya1BXZUJ2L29yVTNMVmJKd0RyU1FiZUthbUNtbwp3cDVicUR4SXdWMjB6cVJiN0FQVU9LWW9WRUZGT0VRczZUNmdJbW5Jb2xoYmlINm00emdaL0NQdldCT2taYytjCjFQbzJFbXZCeitBRDVzQmRUNWt6R1FBNk5iV3laR2xkeFJ0aE5MT3MxZWZPaGRuV0Z1aEkxNjJxbWNmbGdwaUkKV0R1d3E0QzlmK1lrZUpoTm45ZEY1K293bThjT1FtRHJWOE5OZGlUcWluOHEzcVlBSEhKUlcyOGdsSlVDWmtUWgp3SWFTUjZjckJROFRiWU5FMGRjK0NhYTNET0lrejFFT3NIV3pUeCtuMHpLZnFjYmdYaTRESngrQzFianB0WVBSCkJQWkw4REFlV3VBOGVidWRWVDQ0eUVwODJHOTYvR2djZjdGMzN4TXhlMHljK1hhNm93SURBUUFCb0FBd0RRWUoKS29aSWh2Y05BUUVGQlFBRGdnRUJBQjBrY3JGY2NTbUZEbXhveDBOZTAxVUlxU3NEcUhnTCtYbUhUWEp3cmU2RApoSlNad2J2RXRPSzBHMytkcjRGczExV3VVTnQ1cWNMc3g1YTh1azRHNkFLSE16dWhMc0o3WFpqZ21RWEdFQ3BZClE0bUMzeVQzWm9DR3BJWGJ3K2lQM2xtRUVYZ2FRTDBUeDVMRmwvb2tLYktZd0lxTml5S1dPTWo3WlIvd3hXZy8KWkRHUnM1NXh1b2VMREovWlJGZjliSStJYUNVZDFZcmZZY0hJbDNHODdBdityNDlZVndxUkRUMFZEVjd1TGdxbgoyOVhJMVBwVlVOQ1BRR245cC9lWDZRbzd2cERhUHliUnRBMlI3WExLalFhRjlvWFdlQ1VxeTFodkphYzlRRk8yCjk3T2IxYWxwSFBvWjdtV2lFdUp3akJQaWk2YTlNOUczMG5VbzM5bEJpMXc9Ci0tLS0tRU5EIENFUlRJRklDQVRFIFJFUVVFU1QtLS0tLQ=="}`)),
files: make(bootstrap.PathsDataformat),
},
wantErr: false,
},
{
name: "Invalid Old Format",
args: args{
ctx: context.Background(),
data: &bytes.Buffer{},
files: bootstrap.PathsDataformat{
"ServerCA": bootstrap.File{
Timestamp: time.Now(),
},
},
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := migrateBootstrapData(tt.args.ctx, tt.args.data, tt.args.files); (err != nil) != tt.wantErr {
t.Errorf("Cluster.migrateBootstrapData() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestCluster_Snapshot(t *testing.T) {
type fields struct {
clientAccessInfo *clientaccess.Info
config *config.Control
runtime *config.ControlRuntime
managedDB managed.Driver
etcdConfig endpoint.ETCDConfig
joining bool
storageStarted bool
saveBootstrap bool
shouldBootstrap bool
}
type args struct {
ctx context.Context
config *config.Control
}
tests := []struct {
name string
fields fields
args args
wantErr bool
}{
{
name: "Fail on non etcd cluster",
fields: fields{},
args: args{
ctx: context.Background(),
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &Cluster{
clientAccessInfo: tt.fields.clientAccessInfo,
config: tt.fields.config,
runtime: tt.fields.runtime,
managedDB: tt.fields.managedDB,
etcdConfig: tt.fields.etcdConfig,
joining: tt.fields.joining,
storageStarted: tt.fields.storageStarted,
saveBootstrap: tt.fields.saveBootstrap,
shouldBootstrap: tt.fields.shouldBootstrap,
}
if err := c.Snapshot(tt.args.ctx, tt.args.config); (err != nil) != tt.wantErr {
t.Errorf("Cluster.Snapshot() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
...@@ -20,11 +20,11 @@ type Cluster struct { ...@@ -20,11 +20,11 @@ type Cluster struct {
config *config.Control config *config.Control
runtime *config.ControlRuntime runtime *config.ControlRuntime
managedDB managed.Driver managedDB managed.Driver
shouldBootstrap bool
storageStarted bool
etcdConfig endpoint.ETCDConfig etcdConfig endpoint.ETCDConfig
joining bool joining bool
storageStarted bool
saveBootstrap bool saveBootstrap bool
shouldBootstrap bool
} }
// Start creates the dynamic tls listener, http request handler, // Start creates the dynamic tls listener, http request handler,
...@@ -79,24 +79,17 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) { ...@@ -79,24 +79,17 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
return nil, err return nil, err
} }
// if necessary, store bootstrap data to datastore if err := c.startStorage(ctx); err != nil {
if c.saveBootstrap { return nil, err
if err := c.save(ctx); err != nil {
return nil, err
}
} }
// if necessary, record successful bootstrap // if necessary, store bootstrap data to datastore
if c.shouldBootstrap { if c.saveBootstrap {
if err := c.bootstrapped(); err != nil { if err := c.save(ctx, false); err != nil {
return nil, err return nil, err
} }
} }
if err := c.startStorage(ctx); err != nil {
return nil, err
}
// at this point, if etcd is in use, it's bootstrapping is complete // at this point, if etcd is in use, it's bootstrapping is complete
// so save the bootstrap data. We will need for etcd to be up. If // so save the bootstrap data. We will need for etcd to be up. If
// the save call returns an error, we panic since subsequent etcd // the save call returns an error, we panic since subsequent etcd
...@@ -106,7 +99,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) { ...@@ -106,7 +99,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
for { for {
select { select {
case <-ready: case <-ready:
if err := c.save(ctx); err != nil { if err := c.save(ctx, false); err != nil {
panic(err) panic(err)
} }
...@@ -153,7 +146,7 @@ func (c *Cluster) startStorage(ctx context.Context) error { ...@@ -153,7 +146,7 @@ func (c *Cluster) startStorage(ctx context.Context) error {
return nil return nil
} }
// New creates an initial cluster using the provided configuration // New creates an initial cluster using the provided configuration.
func New(config *config.Control) *Cluster { func New(config *config.Control) *Cluster {
return &Cluster{ return &Cluster{
config: config, config: config,
......
...@@ -79,7 +79,8 @@ func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error { ...@@ -79,7 +79,8 @@ func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error {
// Create a HTTP server with the registered request handlers, using logrus for logging // Create a HTTP server with the registered request handlers, using logrus for logging
server := http.Server{ server := http.Server{
Handler: handler} Handler: handler,
}
if logrus.IsLevelEnabled(logrus.DebugLevel) { if logrus.IsLevelEnabled(logrus.DebugLevel) {
server.ErrorLog = log.New(logrus.StandardLogger().Writer(), "Cluster-Http-Server ", log.LstdFlags) server.ErrorLog = log.New(logrus.StandardLogger().Writer(), "Cluster-Http-Server ", log.LstdFlags)
......
...@@ -19,9 +19,9 @@ import ( ...@@ -19,9 +19,9 @@ import (
// snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token. // snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token.
// This is used when bootstrapping a cluster from a managed database or external etcd cluster. // This is used when bootstrapping a cluster from a managed database or external etcd cluster.
// This is NOT used with embedded etcd, which bootstraps over HTTP. // This is NOT used with embedded etcd, which bootstraps over HTTP.
func (c *Cluster) save(ctx context.Context) error { func (c *Cluster) save(ctx context.Context, override bool) error {
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
if err := bootstrap.Write(buf, &c.runtime.ControlRuntimeBootstrap); err != nil { if err := bootstrap.ReadFromDisk(buf, &c.runtime.ControlRuntimeBootstrap); err != nil {
return err return err
} }
token := c.config.Token token := c.config.Token
...@@ -47,14 +47,20 @@ func (c *Cluster) save(ctx context.Context) error { ...@@ -47,14 +47,20 @@ func (c *Cluster) save(ctx context.Context) error {
return err return err
} }
_, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token) if _, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token); err != nil {
if err != nil {
return err return err
} }
if err := storageClient.Create(ctx, storageKey(normalizedToken), data); err != nil { if err := storageClient.Create(ctx, storageKey(normalizedToken), data); err != nil {
if err.Error() == "key exists" { if err.Error() == "key exists" {
logrus.Warnln("bootstrap key already exists") logrus.Warn("bootstrap key already exists")
if override {
bsd, err := c.bootstrapKeyData(ctx, storageClient)
if err != nil {
return err
}
return storageClient.Update(ctx, storageKey(normalizedToken), bsd.Modified, data)
}
return nil return nil
} else if strings.Contains(err.Error(), "not supported for learner") { } else if strings.Contains(err.Error(), "not supported for learner") {
logrus.Debug("skipping bootstrap data save on learner") logrus.Debug("skipping bootstrap data save on learner")
...@@ -66,9 +72,25 @@ func (c *Cluster) save(ctx context.Context) error { ...@@ -66,9 +72,25 @@ func (c *Cluster) save(ctx context.Context) error {
return nil return nil
} }
// bootstrapKeyData lists keys stored in the datastore with the prefix "/bootstrap", and
// will return the first such key. It will return an error if not exactly one key is found.
func (c *Cluster) bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client.Value, error) {
bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0)
if err != nil {
return nil, err
}
if len(bootstrapList) == 0 {
return nil, errors.New("no bootstrap data found")
}
if len(bootstrapList) > 1 {
return nil, errors.New("found multiple bootstrap keys in storage")
}
return &bootstrapList[0], nil
}
// storageBootstrap loads data from the datastore into the ControlRuntimeBootstrap struct. // storageBootstrap loads data from the datastore into the ControlRuntimeBootstrap struct.
// The storage key and encryption passphrase are both derived from the join token. // The storage key and encryption passphrase are both derived from the join token.
// token is either passed // token is either passed.
func (c *Cluster) storageBootstrap(ctx context.Context) error { func (c *Cluster) storageBootstrap(ctx context.Context) error {
if err := c.startStorage(ctx); err != nil { if err := c.startStorage(ctx); err != nil {
return err return err
...@@ -110,7 +132,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { ...@@ -110,7 +132,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
return err return err
} }
return bootstrap.Read(bytes.NewBuffer(data), &c.runtime.ControlRuntimeBootstrap) return c.ReconcileBootstrapData(ctx, bytes.NewReader(data), &c.config.Runtime.ControlRuntimeBootstrap, false, nil)
} }
// getBootstrapKeyFromStorage will list all keys that has prefix /bootstrap and will check for key that is // getBootstrapKeyFromStorage will list all keys that has prefix /bootstrap and will check for key that is
...@@ -157,6 +179,7 @@ func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient ...@@ -157,6 +179,7 @@ func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient
// found then it will still strip the token from any additional info // found then it will still strip the token from any additional info
func readTokenFromFile(serverToken, certs, dataDir string) (string, error) { func readTokenFromFile(serverToken, certs, dataDir string) (string, error) {
tokenFile := filepath.Join(dataDir, "token") tokenFile := filepath.Join(dataDir, "token")
b, err := ioutil.ReadFile(tokenFile) b, err := ioutil.ReadFile(tokenFile)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
...@@ -168,6 +191,7 @@ func readTokenFromFile(serverToken, certs, dataDir string) (string, error) { ...@@ -168,6 +191,7 @@ func readTokenFromFile(serverToken, certs, dataDir string) (string, error) {
} }
return "", err return "", err
} }
// strip the token from any new line if its read from file // strip the token from any new line if its read from file
return string(bytes.TrimRight(b, "\n")), nil return string(bytes.TrimRight(b, "\n")), nil
} }
...@@ -178,6 +202,7 @@ func normalizeToken(token string) (string, error) { ...@@ -178,6 +202,7 @@ func normalizeToken(token string) (string, error) {
if !ok { if !ok {
return password, errors.New("failed to normalize token") return password, errors.New("failed to normalize token")
} }
return password, nil return password, nil
} }
...@@ -186,6 +211,7 @@ func normalizeToken(token string) (string, error) { ...@@ -186,6 +211,7 @@ func normalizeToken(token string) (string, error) {
// then migrate those and resave only with the normalized token // then migrate those and resave only with the normalized token
func (c *Cluster) migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error { func (c *Cluster) migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
oldTokenKey := storageKey(oldToken) oldTokenKey := storageKey(oldToken)
for _, bootstrapKV := range bootstrapList { for _, bootstrapKV := range bootstrapList {
// checking for empty string bootstrap key // checking for empty string bootstrap key
if string(bootstrapKV.Key) == emptyStringKey { if string(bootstrapKV.Key) == emptyStringKey {
...@@ -200,6 +226,7 @@ func (c *Cluster) migrateOldTokens(ctx context.Context, bootstrapList []client.V ...@@ -200,6 +226,7 @@ func (c *Cluster) migrateOldTokens(ctx context.Context, bootstrapList []client.V
} }
} }
} }
return nil return nil
} }
...@@ -209,10 +236,12 @@ func doMigrateToken(ctx context.Context, storageClient client.Client, keyValue c ...@@ -209,10 +236,12 @@ func doMigrateToken(ctx context.Context, storageClient client.Client, keyValue c
if err != nil { if err != nil {
return err return err
} }
encryptedData, err := encrypt(newToken, data) encryptedData, err := encrypt(newToken, data)
if err != nil { if err != nil {
return err return err
} }
// saving the new encrypted data with the right token key // saving the new encrypted data with the right token key
if err := storageClient.Create(ctx, newTokenKey, encryptedData); err != nil { if err := storageClient.Create(ctx, newTokenKey, encryptedData); err != nil {
if err.Error() == "key exists" { if err.Error() == "key exists" {
...@@ -224,10 +253,12 @@ func doMigrateToken(ctx context.Context, storageClient client.Client, keyValue c ...@@ -224,10 +253,12 @@ func doMigrateToken(ctx context.Context, storageClient client.Client, keyValue c
return err return err
} }
} }
logrus.Infof("created bootstrap key %s", newTokenKey) logrus.Infof("created bootstrap key %s", newTokenKey)
// deleting the old key // deleting the old key
if err := storageClient.Delete(ctx, oldTokenKey, keyValue.Modified); err != nil { if err := storageClient.Delete(ctx, oldTokenKey, keyValue.Modified); err != nil {
logrus.Warnf("failed to delete old bootstrap key %s", oldTokenKey) logrus.Warnf("failed to delete old bootstrap key %s", oldTokenKey)
} }
return nil return nil
} }
...@@ -89,9 +89,9 @@ func KubeConfig(dest, url, caCert, clientCert, clientKey string) error { ...@@ -89,9 +89,9 @@ func KubeConfig(dest, url, caCert, clientCert, clientKey string) error {
return kubeconfigTemplate.Execute(output, &data) return kubeconfigTemplate.Execute(output, &data)
} }
// FillRuntimeCerts is responsible for filling out all the // CreateRuntimeCertFiles is responsible for filling out all the
// .crt and .key filenames for a ControlRuntime. // .crt and .key filenames for a ControlRuntime.
func FillRuntimeCerts(config *config.Control, runtime *config.ControlRuntime) { func CreateRuntimeCertFiles(config *config.Control, runtime *config.ControlRuntime) {
runtime.ClientCA = filepath.Join(config.DataDir, "tls", "client-ca.crt") runtime.ClientCA = filepath.Join(config.DataDir, "tls", "client-ca.crt")
runtime.ClientCAKey = filepath.Join(config.DataDir, "tls", "client-ca.key") runtime.ClientCAKey = filepath.Join(config.DataDir, "tls", "client-ca.key")
runtime.ServerCA = filepath.Join(config.DataDir, "tls", "server-ca.crt") runtime.ServerCA = filepath.Join(config.DataDir, "tls", "server-ca.crt")
......
...@@ -239,11 +239,11 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro ...@@ -239,11 +239,11 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
os.MkdirAll(filepath.Join(config.DataDir, "tls"), 0700) os.MkdirAll(filepath.Join(config.DataDir, "tls"), 0700)
os.MkdirAll(filepath.Join(config.DataDir, "cred"), 0700) os.MkdirAll(filepath.Join(config.DataDir, "cred"), 0700)
deps.FillRuntimeCerts(config, runtime) deps.CreateRuntimeCertFiles(config, runtime)
cluster := cluster.New(config) cluster := cluster.New(config)
if err := cluster.Bootstrap(ctx); err != nil { if err := cluster.Bootstrap(ctx, false); err != nil {
return err return err
} }
......
//go:build !no_embedded_executor
// +build !no_embedded_executor // +build !no_embedded_executor
package executor package executor
...@@ -27,6 +28,7 @@ func (e Embedded) ETCD(ctx context.Context, args ETCDConfig) error { ...@@ -27,6 +28,7 @@ func (e Embedded) ETCD(ctx context.Context, args ETCDConfig) error {
if err != nil { if err != nil {
return err return err
} }
etcd, err := embed.StartEtcd(cfg) etcd, err := embed.StartEtcd(cfg)
if err != nil { if err != nil {
return err return err
......
...@@ -149,19 +149,19 @@ func (e *ETCD) Test(ctx context.Context) error { ...@@ -149,19 +149,19 @@ func (e *ETCD) Test(ctx context.Context) error {
return errors.Errorf("this server is a not a member of the etcd cluster. Found %v, expect: %s=%s", memberNameUrls, e.name, e.address) return errors.Errorf("this server is a not a member of the etcd cluster. Found %v, expect: %s=%s", memberNameUrls, e.name, e.address)
} }
// etcdDBDir returns the path to dataDir/db/etcd // DBDir returns the path to dataDir/db/etcd
func etcdDBDir(config *config.Control) string { func DBDir(config *config.Control) string {
return filepath.Join(config.DataDir, "db", "etcd") return filepath.Join(config.DataDir, "db", "etcd")
} }
// walDir returns the path to etcdDBDir/member/wal // walDir returns the path to etcdDBDir/member/wal
func walDir(config *config.Control) string { func walDir(config *config.Control) string {
return filepath.Join(etcdDBDir(config), "member", "wal") return filepath.Join(DBDir(config), "member", "wal")
} }
// nameFile returns the path to etcdDBDir/name. // nameFile returns the path to etcdDBDir/name.
func nameFile(config *config.Control) string { func nameFile(config *config.Control) string {
return filepath.Join(etcdDBDir(config), "name") return filepath.Join(DBDir(config), "name")
} }
// ResetFile returns the path to etcdDBDir/reset-flag. // ResetFile returns the path to etcdDBDir/reset-flag.
...@@ -178,11 +178,11 @@ func (e *ETCD) IsInitialized(ctx context.Context, config *config.Control) (bool, ...@@ -178,11 +178,11 @@ func (e *ETCD) IsInitialized(ctx context.Context, config *config.Control) (bool,
} else if os.IsNotExist(err) { } else if os.IsNotExist(err) {
return false, nil return false, nil
} else { } else {
return false, errors.Wrapf(err, "invalid state for wal directory %s", dir) return false, errors.Wrap(err, "invalid state for wal directory "+dir)
} }
} }
// Reset resets an etcd node // Reset resets an etcd node to a single node cluster.
func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error { func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error {
// Wait for etcd to come up as a new single-node cluster, then exit // Wait for etcd to come up as a new single-node cluster, then exit
go func() { go func() {
...@@ -281,7 +281,7 @@ func (e *ETCD) Start(ctx context.Context, clientAccessInfo *clientaccess.Info) e ...@@ -281,7 +281,7 @@ func (e *ETCD) Start(ctx context.Context, clientAccessInfo *clientaccess.Info) e
if existingCluster { if existingCluster {
//check etcd dir permission //check etcd dir permission
etcdDir := etcdDBDir(e.config) etcdDir := DBDir(e.config)
info, err := os.Stat(etcdDir) info, err := os.Stat(etcdDir)
if err != nil { if err != nil {
return err return err
...@@ -410,10 +410,10 @@ func (e *ETCD) Register(ctx context.Context, config *config.Control, handler htt ...@@ -410,10 +410,10 @@ func (e *ETCD) Register(ctx context.Context, config *config.Control, handler htt
e.config.Datastore.Config.CertFile = e.runtime.ClientETCDCert e.config.Datastore.Config.CertFile = e.runtime.ClientETCDCert
e.config.Datastore.Config.KeyFile = e.runtime.ClientETCDKey e.config.Datastore.Config.KeyFile = e.runtime.ClientETCDKey
tombstoneFile := filepath.Join(etcdDBDir(e.config), "tombstone") tombstoneFile := filepath.Join(DBDir(e.config), "tombstone")
if _, err := os.Stat(tombstoneFile); err == nil { if _, err := os.Stat(tombstoneFile); err == nil {
logrus.Infof("tombstone file has been detected, removing data dir to rejoin the cluster") logrus.Infof("tombstone file has been detected, removing data dir to rejoin the cluster")
if _, err := backupDirWithRetention(etcdDBDir(e.config), maxBackupRetention); err != nil { if _, err := backupDirWithRetention(DBDir(e.config), maxBackupRetention); err != nil {
return nil, err return nil, err
} }
} }
...@@ -503,15 +503,14 @@ func getClientConfig(ctx context.Context, runtime *config.ControlRuntime, endpoi ...@@ -503,15 +503,14 @@ func getClientConfig(ctx context.Context, runtime *config.ControlRuntime, endpoi
if err != nil { if err != nil {
return nil, err return nil, err
} }
cfg := &etcd.Config{ return &etcd.Config{
Endpoints: endpoints, Endpoints: endpoints,
TLS: tlsConfig, TLS: tlsConfig,
Context: ctx, Context: ctx,
DialTimeout: defaultDialTimeout, DialTimeout: defaultDialTimeout,
DialKeepAliveTime: defaultKeepAliveTime, DialKeepAliveTime: defaultKeepAliveTime,
DialKeepAliveTimeout: defaultKeepAliveTimeout, DialKeepAliveTimeout: defaultKeepAliveTimeout,
} }, nil
return cfg, nil
} }
// toTLSConfig converts the ControlRuntime configuration to TLS configuration suitable // toTLSConfig converts the ControlRuntime configuration to TLS configuration suitable
...@@ -589,7 +588,7 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init ...@@ -589,7 +588,7 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init
ListenMetricsURLs: e.metricsURL(e.config.EtcdExposeMetrics), ListenMetricsURLs: e.metricsURL(e.config.EtcdExposeMetrics),
ListenPeerURLs: e.peerURL(), ListenPeerURLs: e.peerURL(),
AdvertiseClientURLs: e.clientURL(), AdvertiseClientURLs: e.clientURL(),
DataDir: etcdDBDir(e.config), DataDir: DBDir(e.config),
ServerTrust: executor.ServerTrust{ ServerTrust: executor.ServerTrust{
CertFile: e.config.Runtime.ServerETCDCert, CertFile: e.config.Runtime.ServerETCDCert,
KeyFile: e.config.Runtime.ServerETCDKey, KeyFile: e.config.Runtime.ServerETCDKey,
...@@ -1248,7 +1247,7 @@ func (e *ETCD) setSnapshotFunction(ctx context.Context) { ...@@ -1248,7 +1247,7 @@ func (e *ETCD) setSnapshotFunction(ctx context.Context) {
// completion. // completion.
func (e *ETCD) Restore(ctx context.Context) error { func (e *ETCD) Restore(ctx context.Context) error {
// check the old etcd data dir // check the old etcd data dir
oldDataDir := etcdDBDir(e.config) + "-old-" + strconv.Itoa(int(time.Now().Unix())) oldDataDir := DBDir(e.config) + "-old-" + strconv.Itoa(int(time.Now().Unix()))
if e.config.ClusterResetRestorePath == "" { if e.config.ClusterResetRestorePath == "" {
return errors.New("no etcd restore path was specified") return errors.New("no etcd restore path was specified")
} }
...@@ -1257,7 +1256,7 @@ func (e *ETCD) Restore(ctx context.Context) error { ...@@ -1257,7 +1256,7 @@ func (e *ETCD) Restore(ctx context.Context) error {
return err return err
} }
// move the data directory to a temp path // move the data directory to a temp path
if err := os.Rename(etcdDBDir(e.config), oldDataDir); err != nil { if err := os.Rename(DBDir(e.config), oldDataDir); err != nil {
return err return err
} }
logrus.Infof("Pre-restore etcd database moved to %s", oldDataDir) logrus.Infof("Pre-restore etcd database moved to %s", oldDataDir)
...@@ -1265,7 +1264,7 @@ func (e *ETCD) Restore(ctx context.Context) error { ...@@ -1265,7 +1264,7 @@ func (e *ETCD) Restore(ctx context.Context) error {
if err := sManager.Restore(snapshot.RestoreConfig{ if err := sManager.Restore(snapshot.RestoreConfig{
SnapshotPath: e.config.ClusterResetRestorePath, SnapshotPath: e.config.ClusterResetRestorePath,
Name: e.name, Name: e.name,
OutputDataDir: etcdDBDir(e.config), OutputDataDir: DBDir(e.config),
OutputWALDir: walDir(e.config), OutputWALDir: walDir(e.config),
PeerURLs: []string{e.peerURL()}, PeerURLs: []string{e.peerURL()},
InitialCluster: e.name + "=" + e.peerURL(), InitialCluster: e.name + "=" + e.peerURL(),
...@@ -1409,8 +1408,8 @@ func (e *ETCD) RemoveSelf(ctx context.Context) error { ...@@ -1409,8 +1408,8 @@ func (e *ETCD) RemoveSelf(ctx context.Context) error {
} }
// backup the data dir to avoid issues when re-enabling etcd // backup the data dir to avoid issues when re-enabling etcd
oldDataDir := etcdDBDir(e.config) + "-old-" + strconv.Itoa(int(time.Now().Unix())) oldDataDir := DBDir(e.config) + "-old-" + strconv.Itoa(int(time.Now().Unix()))
// move the data directory to a temp path // move the data directory to a temp path
return os.Rename(etcdDBDir(e.config), oldDataDir) return os.Rename(DBDir(e.config), oldDataDir)
} }
...@@ -166,17 +166,17 @@ func Test_UnitETCD_Register(t *testing.T) { ...@@ -166,17 +166,17 @@ func Test_UnitETCD_Register(t *testing.T) {
if err := testutil.GenerateRuntime(cnf); err != nil { if err := testutil.GenerateRuntime(cnf); err != nil {
return err return err
} }
if err := os.MkdirAll(etcdDBDir(cnf), 0700); err != nil { if err := os.MkdirAll(DBDir(cnf), 0700); err != nil {
return err return err
} }
tombstoneFile := filepath.Join(etcdDBDir(cnf), "tombstone") tombstoneFile := filepath.Join(DBDir(cnf), "tombstone")
if _, err := os.Create(tombstoneFile); err != nil { if _, err := os.Create(tombstoneFile); err != nil {
return err return err
} }
return nil return nil
}, },
teardown: func(cnf *config.Control) error { teardown: func(cnf *config.Control) error {
tombstoneFile := filepath.Join(etcdDBDir(cnf), "tombstone") tombstoneFile := filepath.Join(DBDir(cnf), "tombstone")
os.Remove(tombstoneFile) os.Remove(tombstoneFile)
testutil.CleanupDataDir(cnf) testutil.CleanupDataDir(cnf)
return nil return nil
......
...@@ -51,7 +51,7 @@ func GenerateRuntime(cnf *config.Control) error { ...@@ -51,7 +51,7 @@ func GenerateRuntime(cnf *config.Control) error {
os.MkdirAll(filepath.Join(cnf.DataDir, "tls"), 0700) os.MkdirAll(filepath.Join(cnf.DataDir, "tls"), 0700)
os.MkdirAll(filepath.Join(cnf.DataDir, "cred"), 0700) os.MkdirAll(filepath.Join(cnf.DataDir, "cred"), 0700)
deps.FillRuntimeCerts(cnf, runtime) deps.CreateRuntimeCertFiles(cnf, runtime)
if err := deps.GenServerDeps(cnf, runtime); err != nil { if err := deps.GenServerDeps(cnf, runtime); err != nil {
return err return err
......
...@@ -13,15 +13,23 @@ type MapClaims map[string]interface{} ...@@ -13,15 +13,23 @@ type MapClaims map[string]interface{}
// Compares the aud claim against cmp. // Compares the aud claim against cmp.
// If required is false, this method will return true if the value matches or is unset // If required is false, this method will return true if the value matches or is unset
func (m MapClaims) VerifyAudience(cmp string, req bool) bool { func (m MapClaims) VerifyAudience(cmp string, req bool) bool {
aud, ok := m["aud"].([]string) var aud []string
if !ok { switch v := m["aud"].(type) {
strAud, ok := m["aud"].(string) case []string:
if !ok { aud = v
return false case []interface{}:
for _, a := range v {
vs, ok := a.(string)
if !ok {
return false
}
aud = append(aud, vs)
} }
aud = append(aud, strAud) case string:
aud = append(aud, v)
default:
return false
} }
return verifyAud(aud, cmp, req) return verifyAud(aud, cmp, req)
} }
......
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package descriptor provides functions for obtaining the protocol buffer
// descriptors of generated Go types.
//
// Deprecated: See the "google.golang.org/protobuf/reflect/protoreflect" package
// for how to obtain an EnumDescriptor or MessageDescriptor in order to
// programatically interact with the protobuf type system.
package descriptor
import (
"bytes"
"compress/gzip"
"io/ioutil"
"sync"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/reflect/protodesc"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/runtime/protoimpl"
descriptorpb "github.com/golang/protobuf/protoc-gen-go/descriptor"
)
// Message is proto.Message with a method to return its descriptor.
//
// Deprecated: The Descriptor method may not be generated by future
// versions of protoc-gen-go, meaning that this interface may not
// be implemented by many concrete message types.
type Message interface {
proto.Message
Descriptor() ([]byte, []int)
}
// ForMessage returns the file descriptor proto containing
// the message and the message descriptor proto for the message itself.
// The returned proto messages must not be mutated.
//
// Deprecated: Not all concrete message types satisfy the Message interface.
// Use MessageDescriptorProto instead. If possible, the calling code should
// be rewritten to use protobuf reflection instead.
// See package "google.golang.org/protobuf/reflect/protoreflect" for details.
func ForMessage(m Message) (*descriptorpb.FileDescriptorProto, *descriptorpb.DescriptorProto) {
return MessageDescriptorProto(m)
}
type rawDesc struct {
fileDesc []byte
indexes []int
}
var rawDescCache sync.Map // map[protoreflect.Descriptor]*rawDesc
func deriveRawDescriptor(d protoreflect.Descriptor) ([]byte, []int) {
// Fast-path: check whether raw descriptors are already cached.
origDesc := d
if v, ok := rawDescCache.Load(origDesc); ok {
return v.(*rawDesc).fileDesc, v.(*rawDesc).indexes
}
// Slow-path: derive the raw descriptor from the v2 descriptor.
// Start with the leaf (a given enum or message declaration) and
// ascend upwards until we hit the parent file descriptor.
var idxs []int
for {
idxs = append(idxs, d.Index())
d = d.Parent()
if d == nil {
// TODO: We could construct a FileDescriptor stub for standalone
// descriptors to satisfy the API.
return nil, nil
}
if _, ok := d.(protoreflect.FileDescriptor); ok {
break
}
}
// Obtain the raw file descriptor.
var raw []byte
switch fd := d.(type) {
case interface{ ProtoLegacyRawDesc() []byte }:
raw = fd.ProtoLegacyRawDesc()
case protoreflect.FileDescriptor:
raw, _ = proto.Marshal(protodesc.ToFileDescriptorProto(fd))
}
file := protoimpl.X.CompressGZIP(raw)
// Reverse the indexes, since we populated it in reverse.
for i, j := 0, len(idxs)-1; i < j; i, j = i+1, j-1 {
idxs[i], idxs[j] = idxs[j], idxs[i]
}
if v, ok := rawDescCache.LoadOrStore(origDesc, &rawDesc{file, idxs}); ok {
return v.(*rawDesc).fileDesc, v.(*rawDesc).indexes
}
return file, idxs
}
// EnumRawDescriptor returns the GZIP'd raw file descriptor representing
// the enum and the index path to reach the enum declaration.
// The returned slices must not be mutated.
func EnumRawDescriptor(e proto.GeneratedEnum) ([]byte, []int) {
if ev, ok := e.(interface{ EnumDescriptor() ([]byte, []int) }); ok {
return ev.EnumDescriptor()
}
ed := protoimpl.X.EnumTypeOf(e)
return deriveRawDescriptor(ed.Descriptor())
}
// MessageRawDescriptor returns the GZIP'd raw file descriptor representing
// the message and the index path to reach the message declaration.
// The returned slices must not be mutated.
func MessageRawDescriptor(m proto.GeneratedMessage) ([]byte, []int) {
if mv, ok := m.(interface{ Descriptor() ([]byte, []int) }); ok {
return mv.Descriptor()
}
md := protoimpl.X.MessageTypeOf(m)
return deriveRawDescriptor(md.Descriptor())
}
var fileDescCache sync.Map // map[*byte]*descriptorpb.FileDescriptorProto
func deriveFileDescriptor(rawDesc []byte) *descriptorpb.FileDescriptorProto {
// Fast-path: check whether descriptor protos are already cached.
if v, ok := fileDescCache.Load(&rawDesc[0]); ok {
return v.(*descriptorpb.FileDescriptorProto)
}
// Slow-path: derive the descriptor proto from the GZIP'd message.
zr, err := gzip.NewReader(bytes.NewReader(rawDesc))
if err != nil {
panic(err)
}
b, err := ioutil.ReadAll(zr)
if err != nil {
panic(err)
}
fd := new(descriptorpb.FileDescriptorProto)
if err := proto.Unmarshal(b, fd); err != nil {
panic(err)
}
if v, ok := fileDescCache.LoadOrStore(&rawDesc[0], fd); ok {
return v.(*descriptorpb.FileDescriptorProto)
}
return fd
}
// EnumDescriptorProto returns the file descriptor proto representing
// the enum and the enum descriptor proto for the enum itself.
// The returned proto messages must not be mutated.
func EnumDescriptorProto(e proto.GeneratedEnum) (*descriptorpb.FileDescriptorProto, *descriptorpb.EnumDescriptorProto) {
rawDesc, idxs := EnumRawDescriptor(e)
if rawDesc == nil || idxs == nil {
return nil, nil
}
fd := deriveFileDescriptor(rawDesc)
if len(idxs) == 1 {
return fd, fd.EnumType[idxs[0]]
}
md := fd.MessageType[idxs[0]]
for _, i := range idxs[1 : len(idxs)-1] {
md = md.NestedType[i]
}
ed := md.EnumType[idxs[len(idxs)-1]]
return fd, ed
}
// MessageDescriptorProto returns the file descriptor proto representing
// the message and the message descriptor proto for the message itself.
// The returned proto messages must not be mutated.
func MessageDescriptorProto(m proto.GeneratedMessage) (*descriptorpb.FileDescriptorProto, *descriptorpb.DescriptorProto) {
rawDesc, idxs := MessageRawDescriptor(m)
if rawDesc == nil || idxs == nil {
return nil, nil
}
fd := deriveFileDescriptor(rawDesc)
md := fd.MessageType[idxs[0]]
for _, i := range idxs[1:] {
md = md.NestedType[i]
}
return fd, md
}
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package remap handles tracking the locations of Go tokens in a source text
// across a rewrite by the Go formatter.
package remap
import (
"fmt"
"go/scanner"
"go/token"
)
// A Location represents a span of byte offsets in the source text.
type Location struct {
Pos, End int // End is exclusive
}
// A Map represents a mapping between token locations in an input source text
// and locations in the correspnding output text.
type Map map[Location]Location
// Find reports whether the specified span is recorded by m, and if so returns
// the new location it was mapped to. If the input span was not found, the
// returned location is the same as the input.
func (m Map) Find(pos, end int) (Location, bool) {
key := Location{
Pos: pos,
End: end,
}
if loc, ok := m[key]; ok {
return loc, true
}
return key, false
}
func (m Map) add(opos, oend, npos, nend int) {
m[Location{Pos: opos, End: oend}] = Location{Pos: npos, End: nend}
}
// Compute constructs a location mapping from input to output. An error is
// reported if any of the tokens of output cannot be mapped.
func Compute(input, output []byte) (Map, error) {
itok := tokenize(input)
otok := tokenize(output)
if len(itok) != len(otok) {
return nil, fmt.Errorf("wrong number of tokens, %d ≠ %d", len(itok), len(otok))
}
m := make(Map)
for i, ti := range itok {
to := otok[i]
if ti.Token != to.Token {
return nil, fmt.Errorf("token %d type mismatch: %s ≠ %s", i+1, ti, to)
}
m.add(ti.pos, ti.end, to.pos, to.end)
}
return m, nil
}
// tokinfo records the span and type of a source token.
type tokinfo struct {
pos, end int
token.Token
}
func tokenize(src []byte) []tokinfo {
fs := token.NewFileSet()
var s scanner.Scanner
s.Init(fs.AddFile("src", fs.Base(), len(src)), src, nil, scanner.ScanComments)
var info []tokinfo
for {
pos, next, lit := s.Scan()
switch next {
case token.SEMICOLON:
continue
}
info = append(info, tokinfo{
pos: int(pos - 1),
end: int(pos + token.Pos(len(lit)) - 1),
Token: next,
})
if next == token.EOF {
break
}
}
return info
}
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: github.com/golang/protobuf/protoc-gen-go/plugin/plugin.proto
package plugin_go
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
pluginpb "google.golang.org/protobuf/types/pluginpb"
reflect "reflect"
)
// Symbols defined in public import of google/protobuf/compiler/plugin.proto.
type CodeGeneratorResponse_Feature = pluginpb.CodeGeneratorResponse_Feature
const CodeGeneratorResponse_FEATURE_NONE = pluginpb.CodeGeneratorResponse_FEATURE_NONE
const CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL = pluginpb.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL
var CodeGeneratorResponse_Feature_name = pluginpb.CodeGeneratorResponse_Feature_name
var CodeGeneratorResponse_Feature_value = pluginpb.CodeGeneratorResponse_Feature_value
type Version = pluginpb.Version
type CodeGeneratorRequest = pluginpb.CodeGeneratorRequest
type CodeGeneratorResponse = pluginpb.CodeGeneratorResponse
type CodeGeneratorResponse_File = pluginpb.CodeGeneratorResponse_File
var File_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto protoreflect.FileDescriptor
var file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_rawDesc = []byte{
0x0a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c,
0x61, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x63, 0x2d, 0x67, 0x65, 0x6e, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69,
0x6e, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x25,
0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f,
0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x75, 0x66, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x2d, 0x67, 0x65, 0x6e, 0x2d, 0x67,
0x6f, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x3b, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x5f,
0x67, 0x6f, 0x50, 0x00, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32,
}
var file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_goTypes = []interface{}{}
var file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_init() }
func file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_init() {
if File_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto != nil {
return
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_rawDesc,
NumEnums: 0,
NumMessages: 0,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_goTypes,
DependencyIndexes: file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_depIdxs,
}.Build()
File_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto = out.File
file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_rawDesc = nil
file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_goTypes = nil
file_github_com_golang_protobuf_protoc_gen_go_plugin_plugin_proto_depIdxs = nil
}
...@@ -796,7 +796,7 @@ func (t *BTree) DescendLessOrEqual(pivot Item, iterator ItemIterator) { ...@@ -796,7 +796,7 @@ func (t *BTree) DescendLessOrEqual(pivot Item, iterator ItemIterator) {
} }
// DescendGreaterThan calls the iterator for every value in the tree within // DescendGreaterThan calls the iterator for every value in the tree within
// the range (pivot, last], until iterator returns false. // the range [last, pivot), until iterator returns false.
func (t *BTree) DescendGreaterThan(pivot Item, iterator ItemIterator) { func (t *BTree) DescendGreaterThan(pivot Item, iterator ItemIterator) {
if t.root == nil { if t.root == nil {
return return
......
// Copyright 2019 The Prometheus Authors // Copyright 2014 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
// You may obtain a copy of the License at // You may obtain a copy of the License at
// //
// http://www.apache.org/licenses/LICENSE-2.0 // http://www.apache.org/licenses/LICENSE-2.0
// //
// Unless required by applicable law or agreed to in writing, software // Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, // distributed under the License is distributed on an "AS IS" BASIS,
...@@ -11,19 +12,6 @@ ...@@ -11,19 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
// +build go1.12 module github.com/google/btree
package prometheus
import "runtime/debug"
// readBuildInfo is a wrapper around debug.ReadBuildInfo for Go 1.12+. go 1.12
func readBuildInfo() (path, version, sum string) {
path, version, sum = "unknown", "unknown", "unknown"
if bi, ok := debug.ReadBuildInfo(); ok {
path = bi.Main.Path
version = bi.Main.Version
sum = bi.Main.Sum
}
return
}
...@@ -200,3 +200,5 @@ coverage.txt ...@@ -200,3 +200,5 @@ coverage.txt
#vendor #vendor
vendor/ vendor/
.envrc
\ No newline at end of file
sudo: false sudo: false
language: go language: go
go: go:
- 1.8.x - 1.13.x
env: - 1.14.x
- DEP_VERSION="0.3.2" - 1.15.x
before_install:
# Download the binary to bin folder in $GOPATH
- curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep
# Make the binary executable
- chmod +x $GOPATH/bin/dep
install: env:
- dep ensure global:
- GO111MODULE=on
script: script:
- make test - make test
after_success: after_success:
- bash <(curl -s https://codecov.io/bash) - bash <(curl -s https://codecov.io/bash)
...@@ -13,10 +13,30 @@ Types of changes: ...@@ -13,10 +13,30 @@ Types of changes:
- `Security` in case of vulnerabilities. - `Security` in case of vulnerabilities.
## [Unreleased] ## [Unreleased]
### Added
- [#223](https://github.com/grpc-ecosystem/go-grpc-middleware/pull/223) Add go-kit logging middleware - [adrien-f](https://github.com/adrien-f)
## [v1.1.0] - 2019-09-12
### Added ### Added
- This CHANGELOG file to keep track of changes. - [#226](https://github.com/grpc-ecosystem/go-grpc-middleware/pull/226) Support for go modules.
- [#221](https://github.com/grpc-ecosystem/go-grpc-middleware/pull/221) logging/zap add support for gRPC LoggerV2 - [kush-patel-hs](https://github.com/kush-patel-hs)
- [#181](https://github.com/grpc-ecosystem/go-grpc-middleware/pull/181) Rate Limit support - [ceshihao](https://github.com/ceshihao)
- [#161](https://github.com/grpc-ecosystem/go-grpc-middleware/pull/161) Retry on server stream call - [lonnblad](https://github.com/lonnblad)
- [#152](https://github.com/grpc-ecosystem/go-grpc-middleware/pull/152) Exponential backoff functions - [polyfloyd](https://github.com/polyfloyd)
- [#147](https://github.com/grpc-ecosystem/go-grpc-middleware/pull/147) Jaeger support for ctxtags extraction - [vporoshok](https://github.com/vporoshok)
- [#184](https://github.com/grpc-ecosystem/go-grpc-middleware/pull/184) ctxTags identifies if the call was sampled
### Deprecated
- [#201](https://github.com/grpc-ecosystem/go-grpc-middleware/pull/201) `golang.org/x/net/context` - [houz42](https://github.com/houz42)
- [#183](https://github.com/grpc-ecosystem/go-grpc-middleware/pull/183) Documentation Generation in favour of <godoc.org>.
### Fixed
- [172](https://github.com/grpc-ecosystem/go-grpc-middleware/pull/172) Passing ctx into retry and recover - [johanbrandhorst](https://github.com/johanbrandhorst)
- Numerious documentation fixes.
## 1.0.0 - 2018-05-08 ## v1.0.0 - 2018-05-08
### Added ### Added
- grpc_auth - grpc_auth
- grpc_ctxtags - grpc_ctxtags
...@@ -27,4 +47,5 @@ Types of changes: ...@@ -27,4 +47,5 @@ Types of changes:
- grpc_validator - grpc_validator
- grpc_recovery - grpc_recovery
[Unreleased]: https://github.com/grpc-ecosystem/go-grpc-middleware/compare/v1.0.0...HEAD [Unreleased]: https://github.com/grpc-ecosystem/go-grpc-middleware/compare/v1.1.0...HEAD
[v1.1.0]: https://github.com/grpc-ecosystem/go-grpc-middleware/compare/v1.0.0...v1.1.0
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "cloud.google.com/go"
packages = ["compute/metadata"]
revision = "2d3a6656c17a60b0815b7e06ab0be04eacb6e613"
version = "v0.16.0"
[[projects]]
name = "github.com/davecgh/go-spew"
packages = ["spew"]
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
name = "github.com/gogo/protobuf"
packages = ["gogoproto","proto","protoc-gen-gogo/descriptor"]
revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02"
version = "v0.5"
[[projects]]
branch = "master"
name = "github.com/golang/protobuf"
packages = ["jsonpb","proto","ptypes","ptypes/any","ptypes/duration","ptypes/struct","ptypes/timestamp"]
revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845"
[[projects]]
name = "github.com/opentracing/opentracing-go"
packages = [".","ext","log","mocktracer"]
revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
version = "v1.0.2"
[[projects]]
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]]
name = "github.com/sirupsen/logrus"
packages = ["."]
revision = "f006c2ac4710855cf0f916dd6b77acf6b048dc6e"
version = "v1.0.3"
[[projects]]
name = "github.com/stretchr/testify"
packages = ["assert","require","suite"]
revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0"
version = "v1.1.4"
[[projects]]
name = "go.uber.org/atomic"
packages = ["."]
revision = "8474b86a5a6f79c443ce4b2992817ff32cf208b8"
version = "v1.3.1"
[[projects]]
name = "go.uber.org/multierr"
packages = ["."]
revision = "3c4937480c32f4c13a875a1829af76c98ca3d40a"
version = "v1.1.0"
[[projects]]
name = "go.uber.org/zap"
packages = [".","buffer","internal/bufferpool","internal/color","internal/exit","zapcore"]
revision = "35aad584952c3e7020db7b839f6b102de6271f89"
version = "v1.7.1"
[[projects]]
branch = "master"
name = "golang.org/x/crypto"
packages = ["ssh/terminal"]
revision = "94eea52f7b742c7cbe0b03b22f0c4c8631ece122"
[[projects]]
branch = "master"
name = "golang.org/x/net"
packages = ["context","context/ctxhttp","http2","http2/hpack","idna","internal/timeseries","lex/httplex","trace"]
revision = "a8b9294777976932365dabb6640cf1468d95c70f"
[[projects]]
branch = "master"
name = "golang.org/x/oauth2"
packages = [".","google","internal","jws","jwt"]
revision = "f95fa95eaa936d9d87489b15d1d18b97c1ba9c28"
[[projects]]
branch = "master"
name = "golang.org/x/sys"
packages = ["unix","windows"]
revision = "13fcbd661c8ececa8807a29b48407d674b1d8ed8"
[[projects]]
branch = "master"
name = "golang.org/x/text"
packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
revision = "75cc3cad82b5f47d3fb229ddda8c5167da14f294"
[[projects]]
name = "google.golang.org/appengine"
packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/urlfetch","urlfetch"]
revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "google.golang.org/genproto"
packages = ["googleapis/rpc/status"]
revision = "7f0da29060c682909f650ad8ed4e515bd74fa12a"
[[projects]]
name = "google.golang.org/grpc"
packages = [".","balancer","balancer/roundrobin","codes","connectivity","credentials","credentials/oauth","encoding","grpclb/grpc_lb_v1/messages","grpclog","internal","keepalive","metadata","naming","peer","resolver","resolver/dns","resolver/passthrough","stats","status","tap","transport"]
revision = "5a9f7b402fe85096d2e1d0383435ee1876e863d0"
version = "v1.8.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "b24c6670412eb0bc44ed1db77fecc52333f8725f3e3272bdc568f5683a63031f"
solver-name = "gps-cdcl"
solver-version = 1
[[constraint]]
name = "github.com/gogo/protobuf"
version = "0.5.0"
[[constraint]]
branch = "master"
name = "github.com/golang/protobuf"
[[constraint]]
name = "github.com/opentracing/opentracing-go"
version = "1.0.2"
[[constraint]]
name = "github.com/sirupsen/logrus"
version = "1.0.3"
[[constraint]]
name = "github.com/stretchr/testify"
version = "1.1.4"
[[constraint]]
name = "go.uber.org/zap"
version = "1.7.1"
[[constraint]]
branch = "master"
name = "golang.org/x/net"
[[constraint]]
branch = "master"
name = "golang.org/x/oauth2"
[[constraint]]
name = "google.golang.org/grpc"
version = "1.8.0"
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
[![codecov](https://codecov.io/gh/grpc-ecosystem/go-grpc-middleware/branch/master/graph/badge.svg)](https://codecov.io/gh/grpc-ecosystem/go-grpc-middleware) [![codecov](https://codecov.io/gh/grpc-ecosystem/go-grpc-middleware/branch/master/graph/badge.svg)](https://codecov.io/gh/grpc-ecosystem/go-grpc-middleware)
[![Apache 2.0 License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) [![Apache 2.0 License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
[![quality: production](https://img.shields.io/badge/quality-production-orange.svg)](#status) [![quality: production](https://img.shields.io/badge/quality-production-orange.svg)](#status)
[![Slack](slack.png)](https://join.slack.com/t/improbable-eng/shared_invite/enQtMzQ1ODcyMzQ5MjM4LWY5ZWZmNGM2ODc5MmViNmQ3ZTA3ZTY3NzQwOTBlMTkzZmIxZTIxODk0OWU3YjZhNWVlNDU3MDlkZGViZjhkMjc) [![Slack](https://img.shields.io/badge/slack-%23grpc--middleware-brightgreen)](https://slack.com/share/IRUQCFC23/9Tm7hxRFVKKNoajQfMOcUiIk/enQtODc4ODI4NTIyMDcxLWM5NDA0ZTE4Njg5YjRjYWZkMTI5MzQwNDY3YzBjMzE1YzdjOGM5ZjI1NDNiM2JmNzI2YjM5ODE5OTRiNTEyOWE)
[gRPC Go](https://github.com/grpc/grpc-go) Middleware: interceptors, helpers, utilities. [gRPC Go](https://github.com/grpc/grpc-go) Middleware: interceptors, helpers, utilities.
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
[gRPC Go](https://github.com/grpc/grpc-go) recently acquired support for [gRPC Go](https://github.com/grpc/grpc-go) recently acquired support for
Interceptors, i.e. [middleware](https://medium.com/@matryer/writing-middleware-in-golang-and-how-go-makes-it-so-much-fun-4375c1246e81#.gv7tdlghs) Interceptors, i.e. [middleware](https://medium.com/@matryer/writing-middleware-in-golang-and-how-go-makes-it-so-much-fun-4375c1246e81#.gv7tdlghs)
that is executed either on the gRPC Server before the request is passed onto the user's application logic, or on the gRPC client either around the user call. It is a perfect way to implement that is executed either on the gRPC Server before the request is passed onto the user's application logic, or on the gRPC client around the user call. It is a perfect way to implement
common patterns: auth, logging, message, validation, retries or monitoring. common patterns: auth, logging, message, validation, retries or monitoring.
These are generic building blocks that make it easy to build multiple microservices easily. These are generic building blocks that make it easy to build multiple microservices easily.
...@@ -29,20 +29,20 @@ import "github.com/grpc-ecosystem/go-grpc-middleware" ...@@ -29,20 +29,20 @@ import "github.com/grpc-ecosystem/go-grpc-middleware"
myServer := grpc.NewServer( myServer := grpc.NewServer(
grpc.StreamInterceptor(grpc_middleware.ChainStreamServer( grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
grpc_recovery.StreamServerInterceptor(),
grpc_ctxtags.StreamServerInterceptor(), grpc_ctxtags.StreamServerInterceptor(),
grpc_opentracing.StreamServerInterceptor(), grpc_opentracing.StreamServerInterceptor(),
grpc_prometheus.StreamServerInterceptor, grpc_prometheus.StreamServerInterceptor,
grpc_zap.StreamServerInterceptor(zapLogger), grpc_zap.StreamServerInterceptor(zapLogger),
grpc_auth.StreamServerInterceptor(myAuthFunction), grpc_auth.StreamServerInterceptor(myAuthFunction),
grpc_recovery.StreamServerInterceptor(),
)), )),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer( grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
grpc_recovery.UnaryServerInterceptor(),
grpc_ctxtags.UnaryServerInterceptor(), grpc_ctxtags.UnaryServerInterceptor(),
grpc_opentracing.UnaryServerInterceptor(), grpc_opentracing.UnaryServerInterceptor(),
grpc_prometheus.UnaryServerInterceptor, grpc_prometheus.UnaryServerInterceptor,
grpc_zap.UnaryServerInterceptor(zapLogger), grpc_zap.UnaryServerInterceptor(zapLogger),
grpc_auth.UnaryServerInterceptor(myAuthFunction), grpc_auth.UnaryServerInterceptor(myAuthFunction),
grpc_recovery.UnaryServerInterceptor(),
)), )),
) )
``` ```
...@@ -58,7 +58,8 @@ myServer := grpc.NewServer( ...@@ -58,7 +58,8 @@ myServer := grpc.NewServer(
* [`grpc_ctxtags`](tags/) - a library that adds a `Tag` map to context, with data populated from request body * [`grpc_ctxtags`](tags/) - a library that adds a `Tag` map to context, with data populated from request body
* [`grpc_zap`](logging/zap/) - integration of [zap](https://github.com/uber-go/zap) logging library into gRPC handlers. * [`grpc_zap`](logging/zap/) - integration of [zap](https://github.com/uber-go/zap) logging library into gRPC handlers.
* [`grpc_logrus`](logging/logrus/) - integration of [logrus](https://github.com/sirupsen/logrus) logging library into gRPC handlers. * [`grpc_logrus`](logging/logrus/) - integration of [logrus](https://github.com/sirupsen/logrus) logging library into gRPC handlers.
* [`grpc_kit`](logging/kit/) - integration of [go-kit](https://github.com/go-kit/kit/tree/master/log) logging library into gRPC handlers.
* [`grpc_grpc_logsettable`](logging/settable/) - a wrapper around `grpclog.LoggerV2` that allows to replace loggers in runtime (thread-safe).
#### Monitoring #### Monitoring
* [`grpc_prometheus`⚡](https://github.com/grpc-ecosystem/go-grpc-prometheus) - Prometheus client-side and server-side monitoring middleware * [`grpc_prometheus`⚡](https://github.com/grpc-ecosystem/go-grpc-prometheus) - Prometheus client-side and server-side monitoring middleware
...@@ -71,6 +72,7 @@ myServer := grpc.NewServer( ...@@ -71,6 +72,7 @@ myServer := grpc.NewServer(
#### Server #### Server
* [`grpc_validator`](validator/) - codegen inbound message validation from `.proto` options * [`grpc_validator`](validator/) - codegen inbound message validation from `.proto` options
* [`grpc_recovery`](recovery/) - turn panics into gRPC errors * [`grpc_recovery`](recovery/) - turn panics into gRPC errors
* [`ratelimit`](ratelimit/) - grpc rate limiting by your own limiter
## Status ## Status
......
...@@ -6,7 +6,8 @@ ...@@ -6,7 +6,8 @@
package grpc_middleware package grpc_middleware
import ( import (
"golang.org/x/net/context" "context"
"google.golang.org/grpc" "google.golang.org/grpc"
) )
...@@ -18,35 +19,19 @@ import ( ...@@ -18,35 +19,19 @@ import (
func ChainUnaryServer(interceptors ...grpc.UnaryServerInterceptor) grpc.UnaryServerInterceptor { func ChainUnaryServer(interceptors ...grpc.UnaryServerInterceptor) grpc.UnaryServerInterceptor {
n := len(interceptors) n := len(interceptors)
if n > 1 { return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
lastI := n - 1 chainer := func(currentInter grpc.UnaryServerInterceptor, currentHandler grpc.UnaryHandler) grpc.UnaryHandler {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { return func(currentCtx context.Context, currentReq interface{}) (interface{}, error) {
var ( return currentInter(currentCtx, currentReq, info, currentHandler)
chainHandler grpc.UnaryHandler
curI int
)
chainHandler = func(currentCtx context.Context, currentReq interface{}) (interface{}, error) {
if curI == lastI {
return handler(currentCtx, currentReq)
}
curI++
resp, err := interceptors[curI](currentCtx, currentReq, info, chainHandler)
curI--
return resp, err
} }
return interceptors[0](ctx, req, info, chainHandler)
} }
}
if n == 1 { chainedHandler := handler
return interceptors[0] for i := n - 1; i >= 0; i-- {
} chainedHandler = chainer(interceptors[i], chainedHandler)
}
// n == 0; Dummy interceptor maintained for backward compatibility to avoid returning nil. return chainedHandler(ctx, req)
return func(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
return handler(ctx, req)
} }
} }
...@@ -58,35 +43,19 @@ func ChainUnaryServer(interceptors ...grpc.UnaryServerInterceptor) grpc.UnarySer ...@@ -58,35 +43,19 @@ func ChainUnaryServer(interceptors ...grpc.UnaryServerInterceptor) grpc.UnarySer
func ChainStreamServer(interceptors ...grpc.StreamServerInterceptor) grpc.StreamServerInterceptor { func ChainStreamServer(interceptors ...grpc.StreamServerInterceptor) grpc.StreamServerInterceptor {
n := len(interceptors) n := len(interceptors)
if n > 1 { return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
lastI := n - 1 chainer := func(currentInter grpc.StreamServerInterceptor, currentHandler grpc.StreamHandler) grpc.StreamHandler {
return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { return func(currentSrv interface{}, currentStream grpc.ServerStream) error {
var ( return currentInter(currentSrv, currentStream, info, currentHandler)
chainHandler grpc.StreamHandler
curI int
)
chainHandler = func(currentSrv interface{}, currentStream grpc.ServerStream) error {
if curI == lastI {
return handler(currentSrv, currentStream)
}
curI++
err := interceptors[curI](currentSrv, currentStream, info, chainHandler)
curI--
return err
} }
return interceptors[0](srv, stream, info, chainHandler)
} }
}
if n == 1 { chainedHandler := handler
return interceptors[0] for i := n - 1; i >= 0; i-- {
} chainedHandler = chainer(interceptors[i], chainedHandler)
}
// n == 0; Dummy interceptor maintained for backward compatibility to avoid returning nil. return chainedHandler(srv, ss)
return func(srv interface{}, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
return handler(srv, stream)
} }
} }
...@@ -97,35 +66,19 @@ func ChainStreamServer(interceptors ...grpc.StreamServerInterceptor) grpc.Stream ...@@ -97,35 +66,19 @@ func ChainStreamServer(interceptors ...grpc.StreamServerInterceptor) grpc.Stream
func ChainUnaryClient(interceptors ...grpc.UnaryClientInterceptor) grpc.UnaryClientInterceptor { func ChainUnaryClient(interceptors ...grpc.UnaryClientInterceptor) grpc.UnaryClientInterceptor {
n := len(interceptors) n := len(interceptors)
if n > 1 { return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
lastI := n - 1 chainer := func(currentInter grpc.UnaryClientInterceptor, currentInvoker grpc.UnaryInvoker) grpc.UnaryInvoker {
return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { return func(currentCtx context.Context, currentMethod string, currentReq, currentRepl interface{}, currentConn *grpc.ClientConn, currentOpts ...grpc.CallOption) error {
var ( return currentInter(currentCtx, currentMethod, currentReq, currentRepl, currentConn, currentInvoker, currentOpts...)
chainHandler grpc.UnaryInvoker
curI int
)
chainHandler = func(currentCtx context.Context, currentMethod string, currentReq, currentRepl interface{}, currentConn *grpc.ClientConn, currentOpts ...grpc.CallOption) error {
if curI == lastI {
return invoker(currentCtx, currentMethod, currentReq, currentRepl, currentConn, currentOpts...)
}
curI++
err := interceptors[curI](currentCtx, currentMethod, currentReq, currentRepl, currentConn, chainHandler, currentOpts...)
curI--
return err
} }
return interceptors[0](ctx, method, req, reply, cc, chainHandler, opts...)
} }
}
if n == 1 { chainedInvoker := invoker
return interceptors[0] for i := n - 1; i >= 0; i-- {
} chainedInvoker = chainer(interceptors[i], chainedInvoker)
}
// n == 0; Dummy interceptor maintained for backward compatibility to avoid returning nil. return chainedInvoker(ctx, method, req, reply, cc, opts...)
return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
return invoker(ctx, method, req, reply, cc, opts...)
} }
} }
...@@ -136,35 +89,19 @@ func ChainUnaryClient(interceptors ...grpc.UnaryClientInterceptor) grpc.UnaryCli ...@@ -136,35 +89,19 @@ func ChainUnaryClient(interceptors ...grpc.UnaryClientInterceptor) grpc.UnaryCli
func ChainStreamClient(interceptors ...grpc.StreamClientInterceptor) grpc.StreamClientInterceptor { func ChainStreamClient(interceptors ...grpc.StreamClientInterceptor) grpc.StreamClientInterceptor {
n := len(interceptors) n := len(interceptors)
if n > 1 { return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
lastI := n - 1 chainer := func(currentInter grpc.StreamClientInterceptor, currentStreamer grpc.Streamer) grpc.Streamer {
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { return func(currentCtx context.Context, currentDesc *grpc.StreamDesc, currentConn *grpc.ClientConn, currentMethod string, currentOpts ...grpc.CallOption) (grpc.ClientStream, error) {
var ( return currentInter(currentCtx, currentDesc, currentConn, currentMethod, currentStreamer, currentOpts...)
chainHandler grpc.Streamer
curI int
)
chainHandler = func(currentCtx context.Context, currentDesc *grpc.StreamDesc, currentConn *grpc.ClientConn, currentMethod string, currentOpts ...grpc.CallOption) (grpc.ClientStream, error) {
if curI == lastI {
return streamer(currentCtx, currentDesc, currentConn, currentMethod, currentOpts...)
}
curI++
stream, err := interceptors[curI](currentCtx, currentDesc, currentConn, currentMethod, chainHandler, currentOpts...)
curI--
return stream, err
} }
return interceptors[0](ctx, desc, cc, method, chainHandler, opts...)
} }
}
if n == 1 { chainedStreamer := streamer
return interceptors[0] for i := n - 1; i >= 0; i-- {
} chainedStreamer = chainer(interceptors[i], chainedStreamer)
}
// n == 0; Dummy interceptor maintained for backward compatibility to avoid returning nil. return chainedStreamer(ctx, desc, cc, method, opts...)
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return streamer(ctx, desc, cc, method, opts...)
} }
} }
......
...@@ -23,7 +23,7 @@ server chaining: ...@@ -23,7 +23,7 @@ server chaining:
myServer := grpc.NewServer( myServer := grpc.NewServer(
grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(loggingStream, monitoringStream, authStream)), grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(loggingStream, monitoringStream, authStream)),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(loggingUnary, monitoringUnary, authUnary), grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(loggingUnary, monitoringUnary, authUnary)),
) )
These interceptors will be executed from left to right: logging, monitoring and auth. These interceptors will be executed from left to right: logging, monitoring and auth.
...@@ -63,7 +63,7 @@ needed. For example: ...@@ -63,7 +63,7 @@ needed. For example:
func FakeAuthStreamingInterceptor(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { func FakeAuthStreamingInterceptor(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
newStream := grpc_middleware.WrapServerStream(stream) newStream := grpc_middleware.WrapServerStream(stream)
newStream.WrappedContext = context.WithValue(ctx, "user_id", "john@example.com") newStream.WrappedContext = context.WithValue(ctx, "user_id", "john@example.com")
return handler(srv, stream) return handler(srv, newStream)
} }
*/ */
package grpc_middleware package grpc_middleware
module github.com/grpc-ecosystem/go-grpc-middleware
require (
github.com/go-kit/kit v0.9.0
github.com/go-logfmt/logfmt v0.4.0 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/gogo/protobuf v1.3.2
github.com/golang/protobuf v1.3.3
github.com/opentracing/opentracing-go v1.1.0
github.com/pkg/errors v0.8.1 // indirect
github.com/sirupsen/logrus v1.4.2
github.com/stretchr/testify v1.4.0
go.uber.org/atomic v1.4.0 // indirect
go.uber.org/multierr v1.1.0 // indirect
go.uber.org/zap v1.10.0
golang.org/x/net v0.0.0-20201021035429-f5854403a974
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215 // indirect
google.golang.org/grpc v1.29.1
)
go 1.14
...@@ -8,7 +8,8 @@ fmt: ...@@ -8,7 +8,8 @@ fmt:
go fmt $(GOFILES_NOVENDOR) go fmt $(GOFILES_NOVENDOR)
vet: vet:
go vet $(GOFILES_NOVENDOR) # do not check lostcancel, they are intentional.
go vet -lostcancel=false $(GOFILES_NOVENDOR)
test: vet test: vet
./scripts/test_all.sh ./scripts/test_all.sh
......
...@@ -4,7 +4,8 @@ ...@@ -4,7 +4,8 @@
package grpc_middleware package grpc_middleware
import ( import (
"golang.org/x/net/context" "context"
"google.golang.org/grpc" "google.golang.org/grpc"
) )
......
load("@rules_proto//proto:defs.bzl", "proto_library")
load("@io_bazel_rules_go//go:def.bzl", "go_library") load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
...@@ -5,7 +6,7 @@ package(default_visibility = ["//visibility:public"]) ...@@ -5,7 +6,7 @@ package(default_visibility = ["//visibility:public"])
proto_library( proto_library(
name = "internal_proto", name = "internal_proto",
srcs = ["stream_chunk.proto"], srcs = ["errors.proto"],
deps = ["@com_google_protobuf//:any_proto"], deps = ["@com_google_protobuf//:any_proto"],
) )
......
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// source: internal/stream_chunk.proto // source: internal/errors.proto
package internal package internal
import proto "github.com/golang/protobuf/proto" import (
import fmt "fmt" fmt "fmt"
import math "math" proto "github.com/golang/protobuf/proto"
import any "github.com/golang/protobuf/ptypes/any" any "github.com/golang/protobuf/ptypes/any"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used. // Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal var _ = proto.Marshal
...@@ -17,7 +19,74 @@ var _ = math.Inf ...@@ -17,7 +19,74 @@ var _ = math.Inf
// is compatible with the proto package it is being compiled against. // is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the // A compilation error at this line likely means your copy of the
// proto package needs to be updated. // proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
// Error is the generic error returned from unary RPCs.
type Error struct {
Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
// This is to make the error more compatible with users that expect errors to be Status objects:
// https://github.com/grpc/grpc/blob/master/src/proto/grpc/status/status.proto
// It should be the exact same message as the Error field.
Code int32 `protobuf:"varint,2,opt,name=code,proto3" json:"code,omitempty"`
Message string `protobuf:"bytes,3,opt,name=message,proto3" json:"message,omitempty"`
Details []*any.Any `protobuf:"bytes,4,rep,name=details,proto3" json:"details,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Error) Reset() { *m = Error{} }
func (m *Error) String() string { return proto.CompactTextString(m) }
func (*Error) ProtoMessage() {}
func (*Error) Descriptor() ([]byte, []int) {
return fileDescriptor_9b093362ca6d1e03, []int{0}
}
func (m *Error) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Error.Unmarshal(m, b)
}
func (m *Error) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Error.Marshal(b, m, deterministic)
}
func (m *Error) XXX_Merge(src proto.Message) {
xxx_messageInfo_Error.Merge(m, src)
}
func (m *Error) XXX_Size() int {
return xxx_messageInfo_Error.Size(m)
}
func (m *Error) XXX_DiscardUnknown() {
xxx_messageInfo_Error.DiscardUnknown(m)
}
var xxx_messageInfo_Error proto.InternalMessageInfo
func (m *Error) GetError() string {
if m != nil {
return m.Error
}
return ""
}
func (m *Error) GetCode() int32 {
if m != nil {
return m.Code
}
return 0
}
func (m *Error) GetMessage() string {
if m != nil {
return m.Message
}
return ""
}
func (m *Error) GetDetails() []*any.Any {
if m != nil {
return m.Details
}
return nil
}
// StreamError is a response type which is returned when // StreamError is a response type which is returned when
// streaming rpc returns an error. // streaming rpc returns an error.
...@@ -36,16 +105,17 @@ func (m *StreamError) Reset() { *m = StreamError{} } ...@@ -36,16 +105,17 @@ func (m *StreamError) Reset() { *m = StreamError{} }
func (m *StreamError) String() string { return proto.CompactTextString(m) } func (m *StreamError) String() string { return proto.CompactTextString(m) }
func (*StreamError) ProtoMessage() {} func (*StreamError) ProtoMessage() {}
func (*StreamError) Descriptor() ([]byte, []int) { func (*StreamError) Descriptor() ([]byte, []int) {
return fileDescriptor_stream_chunk_a2afb657504565d7, []int{0} return fileDescriptor_9b093362ca6d1e03, []int{1}
} }
func (m *StreamError) XXX_Unmarshal(b []byte) error { func (m *StreamError) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_StreamError.Unmarshal(m, b) return xxx_messageInfo_StreamError.Unmarshal(m, b)
} }
func (m *StreamError) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { func (m *StreamError) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_StreamError.Marshal(b, m, deterministic) return xxx_messageInfo_StreamError.Marshal(b, m, deterministic)
} }
func (dst *StreamError) XXX_Merge(src proto.Message) { func (m *StreamError) XXX_Merge(src proto.Message) {
xxx_messageInfo_StreamError.Merge(dst, src) xxx_messageInfo_StreamError.Merge(m, src)
} }
func (m *StreamError) XXX_Size() int { func (m *StreamError) XXX_Size() int {
return xxx_messageInfo_StreamError.Size(m) return xxx_messageInfo_StreamError.Size(m)
...@@ -92,27 +162,28 @@ func (m *StreamError) GetDetails() []*any.Any { ...@@ -92,27 +162,28 @@ func (m *StreamError) GetDetails() []*any.Any {
} }
func init() { func init() {
proto.RegisterType((*Error)(nil), "grpc.gateway.runtime.Error")
proto.RegisterType((*StreamError)(nil), "grpc.gateway.runtime.StreamError") proto.RegisterType((*StreamError)(nil), "grpc.gateway.runtime.StreamError")
} }
func init() { func init() { proto.RegisterFile("internal/errors.proto", fileDescriptor_9b093362ca6d1e03) }
proto.RegisterFile("internal/stream_chunk.proto", fileDescriptor_stream_chunk_a2afb657504565d7)
} var fileDescriptor_9b093362ca6d1e03 = []byte{
// 252 bytes of a gzipped FileDescriptorProto
var fileDescriptor_stream_chunk_a2afb657504565d7 = []byte{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x90, 0xc1, 0x4a, 0xc4, 0x30,
// 223 bytes of a gzipped FileDescriptorProto 0x10, 0x86, 0x89, 0xbb, 0x75, 0xdb, 0xe9, 0x2d, 0x54, 0x88, 0xee, 0xc1, 0xb2, 0xa7, 0x9e, 0x52,
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x34, 0x90, 0x41, 0x4e, 0xc3, 0x30, 0xd0, 0x27, 0xd0, 0xc5, 0x17, 0xe8, 0xde, 0xbc, 0x2c, 0xd9, 0xdd, 0x31, 0x16, 0xda, 0xa4, 0x24,
0x10, 0x45, 0x15, 0x4a, 0x69, 0x3b, 0xd9, 0x45, 0x5d, 0x18, 0xba, 0x20, 0x62, 0x95, 0x95, 0x23, 0x53, 0xa4, 0xf8, 0x56, 0x3e, 0xa1, 0x24, 0xa5, 0xb0, 0x27, 0xf1, 0xd6, 0xf9, 0xfb, 0xcf, 0x7c,
0xc1, 0x09, 0x00, 0x71, 0x81, 0x74, 0xc7, 0xa6, 0x9a, 0x26, 0x83, 0x13, 0x91, 0xd8, 0xd1, 0x78, 0x1f, 0x81, 0xbb, 0xd6, 0x10, 0x3a, 0xa3, 0xba, 0x1a, 0x9d, 0xb3, 0xce, 0xcb, 0xc1, 0x59, 0xb2,
0x22, 0x94, 0x6b, 0x71, 0xc2, 0xca, 0x8e, 0xb2, 0xf4, 0x7b, 0x7f, 0xbe, 0xbe, 0x0c, 0xa7, 0xce, 0xbc, 0xd0, 0x6e, 0x38, 0x4b, 0xad, 0x08, 0xbf, 0xd4, 0x24, 0xdd, 0x68, 0xa8, 0xed, 0xf1, 0xe1,
0x0a, 0xb1, 0xc5, 0xbe, 0xf4, 0xc2, 0x84, 0xc3, 0xa5, 0x6e, 0x27, 0xfb, 0xab, 0x47, 0x76, 0xe2, 0x5e, 0x5b, 0xab, 0x3b, 0xac, 0x63, 0xe7, 0x34, 0x7e, 0xd4, 0xca, 0x4c, 0xf3, 0xc2, 0xee, 0x1b,
0xb2, 0xa3, 0xe1, 0xb1, 0xd6, 0x06, 0x85, 0xfe, 0x70, 0xd6, 0x3c, 0x59, 0xe9, 0x06, 0x7a, 0x7a, 0x92, 0xb7, 0x70, 0x80, 0x17, 0x90, 0xc4, 0x4b, 0x82, 0x95, 0xac, 0xca, 0x9a, 0x79, 0xe0, 0x1c,
0x34, 0xce, 0x99, 0x9e, 0xca, 0x98, 0xb9, 0x4e, 0x3f, 0x25, 0xda, 0x79, 0x39, 0x78, 0xf9, 0x4f, 0xd6, 0x67, 0x7b, 0x41, 0x71, 0x53, 0xb2, 0x2a, 0x69, 0xe2, 0x37, 0x17, 0xb0, 0xe9, 0xd1, 0x7b,
0x20, 0x3d, 0xc7, 0x9e, 0x2f, 0x66, 0xc7, 0xd9, 0x09, 0x0e, 0xa1, 0xe2, 0x52, 0xbb, 0x86, 0x54, 0xa5, 0x51, 0xac, 0x62, 0x77, 0x19, 0xb9, 0x84, 0xcd, 0x05, 0x49, 0xb5, 0x9d, 0x17, 0xeb, 0x72,
0x92, 0x27, 0xc5, 0xb6, 0xda, 0x07, 0xf0, 0xe9, 0x1a, 0x0a, 0xb2, 0x15, 0x19, 0x17, 0x79, 0xb7, 0x55, 0xe5, 0x4f, 0x85, 0x9c, 0xc9, 0x72, 0x21, 0xcb, 0x17, 0x33, 0x35, 0x4b, 0x69, 0xf7, 0xc3,
0xc8, 0x00, 0xa2, 0x54, 0xb0, 0x1b, 0xc8, 0x7b, 0x34, 0xa4, 0x36, 0x79, 0x52, 0x1c, 0xaa, 0xf5, 0x20, 0x3f, 0x90, 0x43, 0xd5, 0xcf, 0x0e, 0x5b, 0xc8, 0x82, 0xff, 0x31, 0x22, 0x59, 0x44, 0xa6,
0x99, 0x3d, 0x43, 0x1a, 0xcf, 0xbc, 0xa0, 0x4c, 0x5e, 0xdd, 0x47, 0x0b, 0x01, 0x9d, 0x23, 0xc9, 0x21, 0xd8, 0x07, 0xec, 0x16, 0xb2, 0x4f, 0xa2, 0xe1, 0x78, 0xe5, 0x93, 0x86, 0x60, 0xff, 0xb7,
0x34, 0xec, 0x1a, 0x12, 0xec, 0x7a, 0xaf, 0xb6, 0xf9, 0xa6, 0x48, 0x5f, 0x8f, 0x7a, 0x59, 0xac, 0xd3, 0x23, 0xe4, 0x71, 0xcd, 0x93, 0xa2, 0x31, 0x78, 0x85, 0xbf, 0x10, 0xa2, 0x43, 0x4c, 0xae,
0xd7, 0xc5, 0xfa, 0xdd, 0xce, 0xd5, 0x1a, 0xfa, 0x80, 0xef, 0xfd, 0xfa, 0x09, 0xd7, 0x87, 0x18, 0xa5, 0x93, 0x7f, 0x48, 0xbf, 0xc2, 0x7b, 0xba, 0xbc, 0xfd, 0xe9, 0x36, 0x56, 0x9e, 0x7f, 0x03,
0x79, 0xbb, 0x05, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x7d, 0xa5, 0x18, 0x17, 0x01, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xde, 0x72, 0x6b, 0x83, 0x8e, 0x01, 0x00, 0x00,
} }
...@@ -4,6 +4,17 @@ option go_package = "internal"; ...@@ -4,6 +4,17 @@ option go_package = "internal";
import "google/protobuf/any.proto"; import "google/protobuf/any.proto";
// Error is the generic error returned from unary RPCs.
message Error {
string error = 1;
// This is to make the error more compatible with users that expect errors to be Status objects:
// https://github.com/grpc/grpc/blob/master/src/proto/grpc/status/status.proto
// It should be the exact same message as the Error field.
int32 code = 2;
string message = 3;
repeated google.protobuf.Any details = 4;
}
// StreamError is a response type which is returned when // StreamError is a response type which is returned when
// streaming rpc returns an error. // streaming rpc returns an error.
message StreamError { message StreamError {
......
...@@ -27,11 +27,12 @@ go_library( ...@@ -27,11 +27,12 @@ go_library(
deps = [ deps = [
"//internal:go_default_library", "//internal:go_default_library",
"//utilities:go_default_library", "//utilities:go_default_library",
"@com_github_golang_protobuf//descriptor:go_default_library_gen",
"@com_github_golang_protobuf//jsonpb:go_default_library_gen", "@com_github_golang_protobuf//jsonpb:go_default_library_gen",
"@com_github_golang_protobuf//proto:go_default_library", "@com_github_golang_protobuf//proto:go_default_library",
"@com_github_golang_protobuf//protoc-gen-go/generator:go_default_library_gen",
"@go_googleapis//google/api:httpbody_go_proto", "@go_googleapis//google/api:httpbody_go_proto",
"@io_bazel_rules_go//proto/wkt:any_go_proto", "@io_bazel_rules_go//proto/wkt:any_go_proto",
"@io_bazel_rules_go//proto/wkt:descriptor_go_proto",
"@io_bazel_rules_go//proto/wkt:duration_go_proto", "@io_bazel_rules_go//proto/wkt:duration_go_proto",
"@io_bazel_rules_go//proto/wkt:field_mask_go_proto", "@io_bazel_rules_go//proto/wkt:field_mask_go_proto",
"@io_bazel_rules_go//proto/wkt:timestamp_go_proto", "@io_bazel_rules_go//proto/wkt:timestamp_go_proto",
...@@ -48,6 +49,7 @@ go_test( ...@@ -48,6 +49,7 @@ go_test(
size = "small", size = "small",
srcs = [ srcs = [
"context_test.go", "context_test.go",
"convert_test.go",
"errors_test.go", "errors_test.go",
"fieldmask_test.go", "fieldmask_test.go",
"handler_test.go", "handler_test.go",
...@@ -62,8 +64,8 @@ go_test( ...@@ -62,8 +64,8 @@ go_test(
], ],
embed = [":go_default_library"], embed = [":go_default_library"],
deps = [ deps = [
"//examples/proto/examplepb:go_default_library",
"//internal:go_default_library", "//internal:go_default_library",
"//runtime/internal/examplepb:go_default_library",
"//utilities:go_default_library", "//utilities:go_default_library",
"@com_github_golang_protobuf//jsonpb:go_default_library_gen", "@com_github_golang_protobuf//jsonpb:go_default_library_gen",
"@com_github_golang_protobuf//proto:go_default_library", "@com_github_golang_protobuf//proto:go_default_library",
...@@ -76,7 +78,6 @@ go_test( ...@@ -76,7 +78,6 @@ go_test(
"@io_bazel_rules_go//proto/wkt:struct_go_proto", "@io_bazel_rules_go//proto/wkt:struct_go_proto",
"@io_bazel_rules_go//proto/wkt:timestamp_go_proto", "@io_bazel_rules_go//proto/wkt:timestamp_go_proto",
"@io_bazel_rules_go//proto/wkt:wrappers_go_proto", "@io_bazel_rules_go//proto/wkt:wrappers_go_proto",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes:go_default_library", "@org_golang_google_grpc//codes:go_default_library",
"@org_golang_google_grpc//metadata:go_default_library", "@org_golang_google_grpc//metadata:go_default_library",
"@org_golang_google_grpc//status:go_default_library", "@org_golang_google_grpc//status:go_default_library",
......
...@@ -9,10 +9,10 @@ import ( ...@@ -9,10 +9,10 @@ import (
"net/textproto" "net/textproto"
"strconv" "strconv"
"strings" "strings"
"sync"
"time" "time"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"google.golang.org/grpc/grpclog"
"google.golang.org/grpc/metadata" "google.golang.org/grpc/metadata"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
) )
...@@ -57,19 +57,45 @@ except that the forwarded destination is not another HTTP service but rather ...@@ -57,19 +57,45 @@ except that the forwarded destination is not another HTTP service but rather
a gRPC service. a gRPC service.
*/ */
func AnnotateContext(ctx context.Context, mux *ServeMux, req *http.Request) (context.Context, error) { func AnnotateContext(ctx context.Context, mux *ServeMux, req *http.Request) (context.Context, error) {
ctx, md, err := annotateContext(ctx, mux, req)
if err != nil {
return nil, err
}
if md == nil {
return ctx, nil
}
return metadata.NewOutgoingContext(ctx, md), nil
}
// AnnotateIncomingContext adds context information such as metadata from the request.
// Attach metadata as incoming context.
func AnnotateIncomingContext(ctx context.Context, mux *ServeMux, req *http.Request) (context.Context, error) {
ctx, md, err := annotateContext(ctx, mux, req)
if err != nil {
return nil, err
}
if md == nil {
return ctx, nil
}
return metadata.NewIncomingContext(ctx, md), nil
}
func annotateContext(ctx context.Context, mux *ServeMux, req *http.Request) (context.Context, metadata.MD, error) {
var pairs []string var pairs []string
timeout := DefaultContextTimeout timeout := DefaultContextTimeout
if tm := req.Header.Get(metadataGrpcTimeout); tm != "" { if tm := req.Header.Get(metadataGrpcTimeout); tm != "" {
var err error var err error
timeout, err = timeoutDecode(tm) timeout, err = timeoutDecode(tm)
if err != nil { if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "invalid grpc-timeout: %s", tm) return nil, nil, status.Errorf(codes.InvalidArgument, "invalid grpc-timeout: %s", tm)
} }
} }
for key, vals := range req.Header { for key, vals := range req.Header {
key = textproto.CanonicalMIMEHeaderKey(key)
for _, val := range vals { for _, val := range vals {
key = textproto.CanonicalMIMEHeaderKey(key)
// For backwards-compatibility, pass through 'authorization' header with no prefix. // For backwards-compatibility, pass through 'authorization' header with no prefix.
if key == "Authorization" { if key == "Authorization" {
pairs = append(pairs, "authorization", val) pairs = append(pairs, "authorization", val)
...@@ -80,7 +106,7 @@ func AnnotateContext(ctx context.Context, mux *ServeMux, req *http.Request) (con ...@@ -80,7 +106,7 @@ func AnnotateContext(ctx context.Context, mux *ServeMux, req *http.Request) (con
if strings.HasSuffix(key, metadataHeaderBinarySuffix) { if strings.HasSuffix(key, metadataHeaderBinarySuffix) {
b, err := decodeBinHeader(val) b, err := decodeBinHeader(val)
if err != nil { if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "invalid binary header %s: %s", key, err) return nil, nil, status.Errorf(codes.InvalidArgument, "invalid binary header %s: %s", key, err)
} }
val = string(b) val = string(b)
...@@ -102,8 +128,6 @@ func AnnotateContext(ctx context.Context, mux *ServeMux, req *http.Request) (con ...@@ -102,8 +128,6 @@ func AnnotateContext(ctx context.Context, mux *ServeMux, req *http.Request) (con
} else { } else {
pairs = append(pairs, strings.ToLower(xForwardedFor), fmt.Sprintf("%s, %s", fwd, remoteIP)) pairs = append(pairs, strings.ToLower(xForwardedFor), fmt.Sprintf("%s, %s", fwd, remoteIP))
} }
} else {
grpclog.Infof("invalid remote addr: %s", addr)
} }
} }
...@@ -111,13 +135,13 @@ func AnnotateContext(ctx context.Context, mux *ServeMux, req *http.Request) (con ...@@ -111,13 +135,13 @@ func AnnotateContext(ctx context.Context, mux *ServeMux, req *http.Request) (con
ctx, _ = context.WithTimeout(ctx, timeout) ctx, _ = context.WithTimeout(ctx, timeout)
} }
if len(pairs) == 0 { if len(pairs) == 0 {
return ctx, nil return ctx, nil, nil
} }
md := metadata.Pairs(pairs...) md := metadata.Pairs(pairs...)
for _, mda := range mux.metadataAnnotators { for _, mda := range mux.metadataAnnotators {
md = metadata.Join(md, mda(ctx, req)) md = metadata.Join(md, mda(ctx, req))
} }
return metadata.NewOutgoingContext(ctx, md), nil return ctx, md, nil
} }
// ServerMetadata consists of metadata sent from gRPC server. // ServerMetadata consists of metadata sent from gRPC server.
...@@ -139,6 +163,63 @@ func ServerMetadataFromContext(ctx context.Context) (md ServerMetadata, ok bool) ...@@ -139,6 +163,63 @@ func ServerMetadataFromContext(ctx context.Context) (md ServerMetadata, ok bool)
return return
} }
// ServerTransportStream implements grpc.ServerTransportStream.
// It should only be used by the generated files to support grpc.SendHeader
// outside of gRPC server use.
type ServerTransportStream struct {
mu sync.Mutex
header metadata.MD
trailer metadata.MD
}
// Method returns the method for the stream.
func (s *ServerTransportStream) Method() string {
return ""
}
// Header returns the header metadata of the stream.
func (s *ServerTransportStream) Header() metadata.MD {
s.mu.Lock()
defer s.mu.Unlock()
return s.header.Copy()
}
// SetHeader sets the header metadata.
func (s *ServerTransportStream) SetHeader(md metadata.MD) error {
if md.Len() == 0 {
return nil
}
s.mu.Lock()
s.header = metadata.Join(s.header, md)
s.mu.Unlock()
return nil
}
// SendHeader sets the header metadata.
func (s *ServerTransportStream) SendHeader(md metadata.MD) error {
return s.SetHeader(md)
}
// Trailer returns the cached trailer metadata.
func (s *ServerTransportStream) Trailer() metadata.MD {
s.mu.Lock()
defer s.mu.Unlock()
return s.trailer.Copy()
}
// SetTrailer sets the trailer metadata.
func (s *ServerTransportStream) SetTrailer(md metadata.MD) error {
if md.Len() == 0 {
return nil
}
s.mu.Lock()
s.trailer = metadata.Join(s.trailer, md)
s.mu.Unlock()
return nil
}
func timeoutDecode(s string) (time.Duration, error) { func timeoutDecode(s string) (time.Duration, error) {
size := len(s) size := len(s)
if size < 2 { if size < 2 {
...@@ -175,7 +256,7 @@ func timeoutUnitToDuration(u uint8) (d time.Duration, ok bool) { ...@@ -175,7 +256,7 @@ func timeoutUnitToDuration(u uint8) (d time.Duration, ok bool) {
} }
// isPermanentHTTPHeader checks whether hdr belongs to the list of // isPermanentHTTPHeader checks whether hdr belongs to the list of
// permenant request headers maintained by IANA. // permanent request headers maintained by IANA.
// http://www.iana.org/assignments/message-headers/message-headers.xml // http://www.iana.org/assignments/message-headers/message-headers.xml
func isPermanentHTTPHeader(hdr string) bool { func isPermanentHTTPHeader(hdr string) bool {
switch hdr { switch hdr {
......
...@@ -206,16 +206,22 @@ func BytesSlice(val, sep string) ([][]byte, error) { ...@@ -206,16 +206,22 @@ func BytesSlice(val, sep string) ([][]byte, error) {
// Timestamp converts the given RFC3339 formatted string into a timestamp.Timestamp. // Timestamp converts the given RFC3339 formatted string into a timestamp.Timestamp.
func Timestamp(val string) (*timestamp.Timestamp, error) { func Timestamp(val string) (*timestamp.Timestamp, error) {
var r *timestamp.Timestamp var r timestamp.Timestamp
err := jsonpb.UnmarshalString(val, r) err := jsonpb.UnmarshalString(val, &r)
return r, err if err != nil {
return nil, err
}
return &r, nil
} }
// Duration converts the given string into a timestamp.Duration. // Duration converts the given string into a timestamp.Duration.
func Duration(val string) (*duration.Duration, error) { func Duration(val string) (*duration.Duration, error) {
var r *duration.Duration var r duration.Duration
err := jsonpb.UnmarshalString(val, r) err := jsonpb.UnmarshalString(val, &r)
return r, err if err != nil {
return nil, err
}
return &r, nil
} }
// Enum converts the given string into an int32 that should be type casted into the // Enum converts the given string into an int32 that should be type casted into the
......
...@@ -4,9 +4,9 @@ import ( ...@@ -4,9 +4,9 @@ import (
"context" "context"
"io" "io"
"net/http" "net/http"
"strings"
"github.com/golang/protobuf/proto" "github.com/grpc-ecosystem/grpc-gateway/internal"
"github.com/golang/protobuf/ptypes/any"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"google.golang.org/grpc/grpclog" "google.golang.org/grpc/grpclog"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
...@@ -58,35 +58,59 @@ func HTTPStatusFromCode(code codes.Code) int { ...@@ -58,35 +58,59 @@ func HTTPStatusFromCode(code codes.Code) int {
} }
var ( var (
// HTTPError replies to the request with the error. // HTTPError replies to the request with an error.
//
// HTTPError is called:
// - From generated per-endpoint gateway handler code, when calling the backend results in an error.
// - From gateway runtime code, when forwarding the response message results in an error.
//
// The default value for HTTPError calls the custom error handler configured on the ServeMux via the
// WithProtoErrorHandler serve option if that option was used, calling GlobalHTTPErrorHandler otherwise.
//
// To customize the error handling of a particular ServeMux instance, use the WithProtoErrorHandler
// serve option.
//
// To customize the error format for all ServeMux instances not using the WithProtoErrorHandler serve
// option, set GlobalHTTPErrorHandler to a custom function.
//
// Setting this variable directly to customize error format is deprecated.
HTTPError = MuxOrGlobalHTTPError
// GlobalHTTPErrorHandler is the HTTPError handler for all ServeMux instances not using the
// WithProtoErrorHandler serve option.
//
// You can set a custom function to this variable to customize error format. // You can set a custom function to this variable to customize error format.
HTTPError = DefaultHTTPError GlobalHTTPErrorHandler = DefaultHTTPError
// OtherErrorHandler handles the following error used by the gateway: StatusMethodNotAllowed StatusNotFound and StatusBadRequest
// OtherErrorHandler handles gateway errors from parsing and routing client requests for all
// ServeMux instances not using the WithProtoErrorHandler serve option.
//
// It returns the following error codes: StatusMethodNotAllowed StatusNotFound StatusBadRequest
//
// To customize parsing and routing error handling of a particular ServeMux instance, use the
// WithProtoErrorHandler serve option.
//
// To customize parsing and routing error handling of all ServeMux instances not using the
// WithProtoErrorHandler serve option, set a custom function to this variable.
OtherErrorHandler = DefaultOtherErrorHandler OtherErrorHandler = DefaultOtherErrorHandler
) )
type errorBody struct { // MuxOrGlobalHTTPError uses the mux-configured error handler, falling back to GlobalErrorHandler.
Error string `protobuf:"bytes,1,name=error" json:"error"` func MuxOrGlobalHTTPError(ctx context.Context, mux *ServeMux, marshaler Marshaler, w http.ResponseWriter, r *http.Request, err error) {
// This is to make the error more compatible with users that expect errors to be Status objects: if mux.protoErrorHandler != nil {
// https://github.com/grpc/grpc/blob/master/src/proto/grpc/status/status.proto mux.protoErrorHandler(ctx, mux, marshaler, w, r, err)
// It should be the exact same message as the Error field. } else {
Message string `protobuf:"bytes,1,name=message" json:"message"` GlobalHTTPErrorHandler(ctx, mux, marshaler, w, r, err)
Code int32 `protobuf:"varint,2,name=code" json:"code"` }
Details []*any.Any `protobuf:"bytes,3,rep,name=details" json:"details,omitempty"`
} }
// Make this also conform to proto.Message for builtin JSONPb Marshaler
func (e *errorBody) Reset() { *e = errorBody{} }
func (e *errorBody) String() string { return proto.CompactTextString(e) }
func (*errorBody) ProtoMessage() {}
// DefaultHTTPError is the default implementation of HTTPError. // DefaultHTTPError is the default implementation of HTTPError.
// If "err" is an error from gRPC system, the function replies with the status code mapped by HTTPStatusFromCode. // If "err" is an error from gRPC system, the function replies with the status code mapped by HTTPStatusFromCode.
// If otherwise, it replies with http.StatusInternalServerError. // If otherwise, it replies with http.StatusInternalServerError.
// //
// The response body returned by this function is a JSON object, // The response body returned by this function is a JSON object,
// which contains a member whose key is "error" and whose value is err.Error(). // which contains a member whose key is "error" and whose value is err.Error().
func DefaultHTTPError(ctx context.Context, mux *ServeMux, marshaler Marshaler, w http.ResponseWriter, _ *http.Request, err error) { func DefaultHTTPError(ctx context.Context, mux *ServeMux, marshaler Marshaler, w http.ResponseWriter, r *http.Request, err error) {
const fallback = `{"error": "failed to marshal error message"}` const fallback = `{"error": "failed to marshal error message"}`
s, ok := status.FromError(err) s, ok := status.FromError(err)
...@@ -95,18 +119,19 @@ func DefaultHTTPError(ctx context.Context, mux *ServeMux, marshaler Marshaler, w ...@@ -95,18 +119,19 @@ func DefaultHTTPError(ctx context.Context, mux *ServeMux, marshaler Marshaler, w
} }
w.Header().Del("Trailer") w.Header().Del("Trailer")
w.Header().Del("Transfer-Encoding")
contentType := marshaler.ContentType() contentType := marshaler.ContentType()
// Check marshaler on run time in order to keep backwards compatability // Check marshaler on run time in order to keep backwards compatibility
// An interface param needs to be added to the ContentType() function on // An interface param needs to be added to the ContentType() function on
// the Marshal interface to be able to remove this check // the Marshal interface to be able to remove this check
if httpBodyMarshaler, ok := marshaler.(*HTTPBodyMarshaler); ok { if typeMarshaler, ok := marshaler.(contentTypeMarshaler); ok {
pb := s.Proto() pb := s.Proto()
contentType = httpBodyMarshaler.ContentTypeFromMessage(pb) contentType = typeMarshaler.ContentTypeFromMessage(pb)
} }
w.Header().Set("Content-Type", contentType) w.Header().Set("Content-Type", contentType)
body := &errorBody{ body := &internal.Error{
Error: s.Message(), Error: s.Message(),
Message: s.Message(), Message: s.Message(),
Code: int32(s.Code()), Code: int32(s.Code()),
...@@ -129,14 +154,29 @@ func DefaultHTTPError(ctx context.Context, mux *ServeMux, marshaler Marshaler, w ...@@ -129,14 +154,29 @@ func DefaultHTTPError(ctx context.Context, mux *ServeMux, marshaler Marshaler, w
} }
handleForwardResponseServerMetadata(w, mux, md) handleForwardResponseServerMetadata(w, mux, md)
handleForwardResponseTrailerHeader(w, md)
// RFC 7230 https://tools.ietf.org/html/rfc7230#section-4.1.2
// Unless the request includes a TE header field indicating "trailers"
// is acceptable, as described in Section 4.3, a server SHOULD NOT
// generate trailer fields that it believes are necessary for the user
// agent to receive.
var wantsTrailers bool
if te := r.Header.Get("TE"); strings.Contains(strings.ToLower(te), "trailers") {
wantsTrailers = true
handleForwardResponseTrailerHeader(w, md)
w.Header().Set("Transfer-Encoding", "chunked")
}
st := HTTPStatusFromCode(s.Code()) st := HTTPStatusFromCode(s.Code())
w.WriteHeader(st) w.WriteHeader(st)
if _, err := w.Write(buf); err != nil { if _, err := w.Write(buf); err != nil {
grpclog.Infof("Failed to write response: %v", err) grpclog.Infof("Failed to write response: %v", err)
} }
handleForwardResponseTrailer(w, md) if wantsTrailers {
handleForwardResponseTrailer(w, md)
}
} }
// DefaultOtherErrorHandler is the default implementation of OtherErrorHandler. // DefaultOtherErrorHandler is the default implementation of OtherErrorHandler.
......
...@@ -5,12 +5,37 @@ import ( ...@@ -5,12 +5,37 @@ import (
"io" "io"
"strings" "strings"
"github.com/golang/protobuf/protoc-gen-go/generator" descriptor2 "github.com/golang/protobuf/descriptor"
"github.com/golang/protobuf/protoc-gen-go/descriptor"
"google.golang.org/genproto/protobuf/field_mask" "google.golang.org/genproto/protobuf/field_mask"
) )
func translateName(name string, md *descriptor.DescriptorProto) (string, *descriptor.DescriptorProto) {
// TODO - should really gate this with a test that the marshaller has used json names
if md != nil {
for _, f := range md.Field {
if f.JsonName != nil && f.Name != nil && *f.JsonName == name {
var subType *descriptor.DescriptorProto
// If the field has a TypeName then we retrieve the nested type for translating the embedded message names.
if f.TypeName != nil {
typeSplit := strings.Split(*f.TypeName, ".")
typeName := typeSplit[len(typeSplit)-1]
for _, t := range md.NestedType {
if typeName == *t.Name {
subType = t
}
}
}
return *f.Name, subType
}
}
}
return name, nil
}
// FieldMaskFromRequestBody creates a FieldMask printing all complete paths from the JSON body. // FieldMaskFromRequestBody creates a FieldMask printing all complete paths from the JSON body.
func FieldMaskFromRequestBody(r io.Reader) (*field_mask.FieldMask, error) { func FieldMaskFromRequestBody(r io.Reader, md *descriptor.DescriptorProto) (*field_mask.FieldMask, error) {
fm := &field_mask.FieldMask{} fm := &field_mask.FieldMask{}
var root interface{} var root interface{}
if err := json.NewDecoder(r).Decode(&root); err != nil { if err := json.NewDecoder(r).Decode(&root); err != nil {
...@@ -20,7 +45,7 @@ func FieldMaskFromRequestBody(r io.Reader) (*field_mask.FieldMask, error) { ...@@ -20,7 +45,7 @@ func FieldMaskFromRequestBody(r io.Reader) (*field_mask.FieldMask, error) {
return nil, err return nil, err
} }
queue := []fieldMaskPathItem{{node: root}} queue := []fieldMaskPathItem{{node: root, md: md}}
for len(queue) > 0 { for len(queue) > 0 {
// dequeue an item // dequeue an item
item := queue[0] item := queue[0]
...@@ -29,11 +54,22 @@ func FieldMaskFromRequestBody(r io.Reader) (*field_mask.FieldMask, error) { ...@@ -29,11 +54,22 @@ func FieldMaskFromRequestBody(r io.Reader) (*field_mask.FieldMask, error) {
if m, ok := item.node.(map[string]interface{}); ok { if m, ok := item.node.(map[string]interface{}); ok {
// if the item is an object, then enqueue all of its children // if the item is an object, then enqueue all of its children
for k, v := range m { for k, v := range m {
queue = append(queue, fieldMaskPathItem{path: append(item.path, generator.CamelCase(k)), node: v}) protoName, subMd := translateName(k, item.md)
if subMsg, ok := v.(descriptor2.Message); ok {
_, subMd = descriptor2.ForMessage(subMsg)
}
var path string
if item.path == "" {
path = protoName
} else {
path = item.path + "." + protoName
}
queue = append(queue, fieldMaskPathItem{path: path, node: v, md: subMd})
} }
} else if len(item.path) > 0 { } else if len(item.path) > 0 {
// otherwise, it's a leaf node so print its path // otherwise, it's a leaf node so print its path
fm.Paths = append(fm.Paths, strings.Join(item.path, ".")) fm.Paths = append(fm.Paths, item.path)
} }
} }
...@@ -42,29 +78,12 @@ func FieldMaskFromRequestBody(r io.Reader) (*field_mask.FieldMask, error) { ...@@ -42,29 +78,12 @@ func FieldMaskFromRequestBody(r io.Reader) (*field_mask.FieldMask, error) {
// fieldMaskPathItem stores a in-progress deconstruction of a path for a fieldmask // fieldMaskPathItem stores a in-progress deconstruction of a path for a fieldmask
type fieldMaskPathItem struct { type fieldMaskPathItem struct {
// the list of prior fields leading up to node // the list of prior fields leading up to node connected by dots
path []string path string
// a generic decoded json object the current item to inspect for further path extraction // a generic decoded json object the current item to inspect for further path extraction
node interface{} node interface{}
}
// CamelCaseFieldMask updates the given FieldMask by converting all of its paths to CamelCase, using the same heuristic
// that's used for naming protobuf fields in Go.
func CamelCaseFieldMask(mask *field_mask.FieldMask) {
if mask == nil || mask.Paths == nil {
return
}
var newPaths []string
for _, path := range mask.Paths {
lowerCasedParts := strings.Split(path, ".")
var camelCasedParts []string
for _, part := range lowerCasedParts {
camelCasedParts = append(camelCasedParts, generator.CamelCase(part))
}
newPaths = append(newPaths, strings.Join(camelCasedParts, "."))
}
mask.Paths = newPaths // descriptor for parent message
md *descriptor.DescriptorProto
} }
package runtime package runtime
import ( import (
"context"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"net/textproto" "net/textproto"
"context"
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/grpc-ecosystem/grpc-gateway/internal" "github.com/grpc-ecosystem/grpc-gateway/internal"
"google.golang.org/grpc/grpclog" "google.golang.org/grpc/grpclog"
...@@ -61,7 +61,19 @@ func ForwardResponseStream(ctx context.Context, mux *ServeMux, marshaler Marshal ...@@ -61,7 +61,19 @@ func ForwardResponseStream(ctx context.Context, mux *ServeMux, marshaler Marshal
return return
} }
buf, err := marshaler.Marshal(streamChunk(ctx, resp, mux.streamErrorHandler)) var buf []byte
switch {
case resp == nil:
buf, err = marshaler.Marshal(errorChunk(streamError(ctx, mux.streamErrorHandler, errEmptyResponse)))
default:
result := map[string]interface{}{"result": resp}
if rb, ok := resp.(responseBody); ok {
result["result"] = rb.XXX_ResponseBody()
}
buf, err = marshaler.Marshal(result)
}
if err != nil { if err != nil {
grpclog.Infof("Failed to marshal response chunk: %v", err) grpclog.Infof("Failed to marshal response chunk: %v", err)
handleForwardResponseStreamError(ctx, wroteHeader, marshaler, w, req, mux, err) handleForwardResponseStreamError(ctx, wroteHeader, marshaler, w, req, mux, err)
...@@ -123,11 +135,11 @@ func ForwardResponseMessage(ctx context.Context, mux *ServeMux, marshaler Marsha ...@@ -123,11 +135,11 @@ func ForwardResponseMessage(ctx context.Context, mux *ServeMux, marshaler Marsha
handleForwardResponseTrailerHeader(w, md) handleForwardResponseTrailerHeader(w, md)
contentType := marshaler.ContentType() contentType := marshaler.ContentType()
// Check marshaler on run time in order to keep backwards compatability // Check marshaler on run time in order to keep backwards compatibility
// An interface param needs to be added to the ContentType() function on // An interface param needs to be added to the ContentType() function on
// the Marshal interface to be able to remove this check // the Marshal interface to be able to remove this check
if httpBodyMarshaler, ok := marshaler.(*HTTPBodyMarshaler); ok { if typeMarshaler, ok := marshaler.(contentTypeMarshaler); ok {
contentType = httpBodyMarshaler.ContentTypeFromMessage(resp) contentType = typeMarshaler.ContentTypeFromMessage(resp)
} }
w.Header().Set("Content-Type", contentType) w.Header().Set("Content-Type", contentType)
...@@ -184,15 +196,6 @@ func handleForwardResponseStreamError(ctx context.Context, wroteHeader bool, mar ...@@ -184,15 +196,6 @@ func handleForwardResponseStreamError(ctx context.Context, wroteHeader bool, mar
} }
} }
// streamChunk returns a chunk in a response stream for the given result. The
// given errHandler is used to render an error chunk if result is nil.
func streamChunk(ctx context.Context, result proto.Message, errHandler StreamErrorHandlerFunc) map[string]proto.Message {
if result == nil {
return errorChunk(streamError(ctx, errHandler, errEmptyResponse))
}
return map[string]proto.Message{"result": result}
}
// streamError returns the payload for the final message in a response stream // streamError returns the payload for the final message in a response stream
// that represents the given err. // that represents the given err.
func streamError(ctx context.Context, errHandler StreamErrorHandlerFunc, err error) *StreamError { func streamError(ctx context.Context, errHandler StreamErrorHandlerFunc, err error) *StreamError {
......
...@@ -19,7 +19,7 @@ type HTTPBodyMarshaler struct { ...@@ -19,7 +19,7 @@ type HTTPBodyMarshaler struct {
Marshaler Marshaler
} }
// ContentType implementation to keep backwards compatability with marshal interface // ContentType implementation to keep backwards compatibility with marshal interface
func (h *HTTPBodyMarshaler) ContentType() string { func (h *HTTPBodyMarshaler) ContentType() string {
return h.ContentTypeFromMessage(nil) return h.ContentTypeFromMessage(nil)
} }
......
...@@ -19,6 +19,13 @@ type Marshaler interface { ...@@ -19,6 +19,13 @@ type Marshaler interface {
ContentType() string ContentType() string
} }
// Marshalers that implement contentTypeMarshaler will have their ContentTypeFromMessage method called
// to set the Content-Type header on the response
type contentTypeMarshaler interface {
// ContentTypeFromMessage returns the Content-Type this marshaler produces from the provided message
ContentTypeFromMessage(v interface{}) string
}
// Decoder decodes a byte sequence // Decoder decodes a byte sequence
type Decoder interface { type Decoder interface {
Decode(v interface{}) error Decode(v interface{}) error
...@@ -43,6 +50,6 @@ func (f EncoderFunc) Encode(v interface{}) error { return f(v) } ...@@ -43,6 +50,6 @@ func (f EncoderFunc) Encode(v interface{}) error { return f(v) }
// Delimited defines the streaming delimiter. // Delimited defines the streaming delimiter.
type Delimited interface { type Delimited interface {
// Delimiter returns the record seperator for the stream. // Delimiter returns the record separator for the stream.
Delimiter() []byte Delimiter() []byte
} }
...@@ -2,7 +2,10 @@ package runtime ...@@ -2,7 +2,10 @@ package runtime
import ( import (
"errors" "errors"
"mime"
"net/http" "net/http"
"google.golang.org/grpc/grpclog"
) )
// MIMEWildcard is the fallback MIME type used for requests which do not match // MIMEWildcard is the fallback MIME type used for requests which do not match
...@@ -31,7 +34,12 @@ func MarshalerForRequest(mux *ServeMux, r *http.Request) (inbound Marshaler, out ...@@ -31,7 +34,12 @@ func MarshalerForRequest(mux *ServeMux, r *http.Request) (inbound Marshaler, out
} }
for _, contentTypeVal := range r.Header[contentTypeHeader] { for _, contentTypeVal := range r.Header[contentTypeHeader] {
if m, ok := mux.marshalers.mimeMap[contentTypeVal]; ok { contentType, _, err := mime.ParseMediaType(contentTypeVal)
if err != nil {
grpclog.Infof("Failed to parse Content-Type %s: %v", contentTypeVal, err)
continue
}
if m, ok := mux.marshalers.mimeMap[contentType]; ok {
inbound = m inbound = m
break break
} }
......
...@@ -55,6 +55,15 @@ func WithForwardResponseOption(forwardResponseOption func(context.Context, http. ...@@ -55,6 +55,15 @@ func WithForwardResponseOption(forwardResponseOption func(context.Context, http.
} }
} }
// SetQueryParameterParser sets the query parameter parser, used to populate message from query parameters.
// Configuring this will mean the generated swagger output is no longer correct, and it should be
// done with careful consideration.
func SetQueryParameterParser(queryParameterParser QueryParameterParser) ServeMuxOption {
return func(serveMux *ServeMux) {
currentQueryParser = queryParameterParser
}
}
// HeaderMatcherFunc checks whether a header key should be forwarded to/from gRPC context. // HeaderMatcherFunc checks whether a header key should be forwarded to/from gRPC context.
type HeaderMatcherFunc func(string) (string, bool) type HeaderMatcherFunc func(string) (string, bool)
...@@ -102,11 +111,11 @@ func WithMetadata(annotator func(context.Context, *http.Request) metadata.MD) Se ...@@ -102,11 +111,11 @@ func WithMetadata(annotator func(context.Context, *http.Request) metadata.MD) Se
} }
} }
// WithProtoErrorHandler returns a ServeMuxOption for passing metadata to a gRPC context. // WithProtoErrorHandler returns a ServeMuxOption for configuring a custom error handler.
// //
// This can be used to handle an error as general proto message defined by gRPC. // This can be used to handle an error as general proto message defined by gRPC.
// The response including body and status is not backward compatible with the default error handler. // When this option is used, the mux uses the configured error handler instead of HTTPError and
// When this option is used, HTTPError and OtherErrorHandler are overwritten on initialization. // OtherErrorHandler.
func WithProtoErrorHandler(fn ProtoErrorHandlerFunc) ServeMuxOption { func WithProtoErrorHandler(fn ProtoErrorHandlerFunc) ServeMuxOption {
return func(serveMux *ServeMux) { return func(serveMux *ServeMux) {
serveMux.protoErrorHandler = fn serveMux.protoErrorHandler = fn
...@@ -156,18 +165,6 @@ func NewServeMux(opts ...ServeMuxOption) *ServeMux { ...@@ -156,18 +165,6 @@ func NewServeMux(opts ...ServeMuxOption) *ServeMux {
opt(serveMux) opt(serveMux)
} }
if serveMux.protoErrorHandler != nil {
HTTPError = serveMux.protoErrorHandler
// OtherErrorHandler is no longer used when protoErrorHandler is set.
// Overwritten by a special error handler to return Unknown.
OtherErrorHandler = func(w http.ResponseWriter, r *http.Request, _ string, _ int) {
ctx := context.Background()
_, outboundMarshaler := MarshalerForRequest(serveMux, r)
sterr := status.Error(codes.Unknown, "unexpected use of OtherErrorHandler")
serveMux.protoErrorHandler(ctx, serveMux, outboundMarshaler, w, r, sterr)
}
}
if serveMux.incomingHeaderMatcher == nil { if serveMux.incomingHeaderMatcher == nil {
serveMux.incomingHeaderMatcher = DefaultHeaderMatcher serveMux.incomingHeaderMatcher = DefaultHeaderMatcher
} }
......
...@@ -44,12 +44,12 @@ func DefaultHTTPProtoErrorHandler(ctx context.Context, mux *ServeMux, marshaler ...@@ -44,12 +44,12 @@ func DefaultHTTPProtoErrorHandler(ctx context.Context, mux *ServeMux, marshaler
w.Header().Del("Trailer") w.Header().Del("Trailer")
contentType := marshaler.ContentType() contentType := marshaler.ContentType()
// Check marshaler on run time in order to keep backwards compatability // Check marshaler on run time in order to keep backwards compatibility
// An interface param needs to be added to the ContentType() function on // An interface param needs to be added to the ContentType() function on
// the Marshal interface to be able to remove this check // the Marshal interface to be able to remove this check
if httpBodyMarshaler, ok := marshaler.(*HTTPBodyMarshaler); ok { if typeMarshaler, ok := marshaler.(contentTypeMarshaler); ok {
pb := s.Proto() pb := s.Proto()
contentType = httpBodyMarshaler.ContentTypeFromMessage(pb) contentType = typeMarshaler.ContentTypeFromMessage(pb)
} }
w.Header().Set("Content-Type", contentType) w.Header().Set("Content-Type", contentType)
......
...@@ -15,15 +15,28 @@ import ( ...@@ -15,15 +15,28 @@ import (
"google.golang.org/grpc/grpclog" "google.golang.org/grpc/grpclog"
) )
// PopulateQueryParameters populates "values" into "msg". var valuesKeyRegexp = regexp.MustCompile("^(.*)\\[(.*)\\]$")
// A value is ignored if its key starts with one of the elements in "filter".
var currentQueryParser QueryParameterParser = &defaultQueryParser{}
// QueryParameterParser defines interface for all query parameter parsers
type QueryParameterParser interface {
Parse(msg proto.Message, values url.Values, filter *utilities.DoubleArray) error
}
// PopulateQueryParameters parses query parameters
// into "msg" using current query parser
func PopulateQueryParameters(msg proto.Message, values url.Values, filter *utilities.DoubleArray) error { func PopulateQueryParameters(msg proto.Message, values url.Values, filter *utilities.DoubleArray) error {
return currentQueryParser.Parse(msg, values, filter)
}
type defaultQueryParser struct{}
// Parse populates "values" into "msg".
// A value is ignored if its key starts with one of the elements in "filter".
func (*defaultQueryParser) Parse(msg proto.Message, values url.Values, filter *utilities.DoubleArray) error {
for key, values := range values { for key, values := range values {
re, err := regexp.Compile("^(.*)\\[(.*)\\]$") match := valuesKeyRegexp.FindStringSubmatch(key)
if err != nil {
return err
}
match := re.FindStringSubmatch(key)
if len(match) == 3 { if len(match) == 3 {
key = match[1] key = match[1]
values = append([]string{match[2]}, values...) values = append([]string{match[2]}, values...)
...@@ -119,14 +132,16 @@ func fieldByProtoName(m reflect.Value, name string) (reflect.Value, *proto.Prope ...@@ -119,14 +132,16 @@ func fieldByProtoName(m reflect.Value, name string) (reflect.Value, *proto.Prope
props := proto.GetProperties(m.Type()) props := proto.GetProperties(m.Type())
// look up field name in oneof map // look up field name in oneof map
if op, ok := props.OneofTypes[name]; ok { for _, op := range props.OneofTypes {
v := reflect.New(op.Type.Elem()) if name == op.Prop.OrigName || name == op.Prop.JSONName {
field := m.Field(op.Field) v := reflect.New(op.Type.Elem())
if !field.IsNil() { field := m.Field(op.Field)
return reflect.Value{}, nil, fmt.Errorf("field already set for %s oneof", props.Prop[op.Field].OrigName) if !field.IsNil() {
return reflect.Value{}, nil, fmt.Errorf("field already set for %s oneof", props.Prop[op.Field].OrigName)
}
field.Set(v)
return v.Elem().Field(0), op.Prop, nil
} }
field.Set(v)
return v.Elem().Field(0), op.Prop, nil
} }
for _, p := range props.Prop { for _, p := range props.Prop {
......
root = true
[*]
charset = utf-8
end_of_line = lf
indent_size = 4
indent_style = space
insert_final_newline = true
trim_trailing_whitespace = true
[*.go]
indent_style = tab
/.idea/
# Compiled Object files, Static and Dynamic libs (Shared Objects) # Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o *.o
*.a *.a
......
clockwork # clockwork
=========
[![Build Status](https://travis-ci.org/jonboulle/clockwork.png?branch=master)](https://travis-ci.org/jonboulle/clockwork) [![Mentioned in Awesome Go](https://awesome.re/mentioned-badge-flat.svg)](https://github.com/avelino/awesome-go#utilities)
[![godoc](https://godoc.org/github.com/jonboulle/clockwork?status.svg)](http://godoc.org/github.com/jonboulle/clockwork)
a simple fake clock for golang [![GitHub Workflow Status](https://img.shields.io/github/workflow/status/jonboulle/clockwork/CI?style=flat-square)](https://github.com/jonboulle/clockwork/actions?query=workflow%3ACI)
[![Go Report Card](https://goreportcard.com/badge/github.com/jonboulle/clockwork?style=flat-square)](https://goreportcard.com/report/github.com/jonboulle/clockwork)
![Go Version](https://img.shields.io/badge/go%20version-%3E=1.11-61CFDD.svg?style=flat-square)
[![go.dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/mod/github.com/jonboulle/clockwork)
# Usage **A simple fake clock for Go.**
## Usage
Replace uses of the `time` package with the `clockwork.Clock` interface instead. Replace uses of the `time` package with the `clockwork.Clock` interface instead.
For example, instead of using `time.Sleep` directly: For example, instead of using `time.Sleep` directly:
``` ```go
func my_func() { func myFunc() {
time.Sleep(3 * time.Second) time.Sleep(3 * time.Second)
do_something() doSomething()
} }
``` ```
inject a clock and use its `Sleep` method instead: Inject a clock and use its `Sleep` method instead:
``` ```go
func my_func(clock clockwork.Clock) { func myFunc(clock clockwork.Clock) {
clock.Sleep(3 * time.Second) clock.Sleep(3 * time.Second)
do_something() doSomething()
} }
``` ```
Now you can easily test `my_func` with a `FakeClock`: Now you can easily test `myFunc` with a `FakeClock`:
``` ```go
func TestMyFunc(t *testing.T) { func TestMyFunc(t *testing.T) {
c := clockwork.NewFakeClock() c := clockwork.NewFakeClock()
// Start our sleepy function // Start our sleepy function
my_func(c) var wg sync.WaitGroup
wg.Add(1)
// Ensure we wait until my_func is sleeping go func() {
myFunc(c)
wg.Done()
}()
// Ensure we wait until myFunc is sleeping
c.BlockUntil(1) c.BlockUntil(1)
assert_state() assertState()
// Advance the FakeClock forward in time // Advance the FakeClock forward in time
c.Advance(3) c.Advance(3 * time.Second)
// Wait until the function completes
wg.Wait()
assert_state() assertState()
} }
``` ```
and in production builds, simply inject the real clock instead: and in production builds, simply inject the real clock instead:
```
my_func(clockwork.NewRealClock()) ```go
myFunc(clockwork.NewRealClock())
``` ```
See [example_test.go](example_test.go) for a full example. See [example_test.go](example_test.go) for a full example.
# Credits # Credits
clockwork is inspired by @wickman's [threaded fake clock](https://gist.github.com/wickman/3840816), and the [Golang playground](http://blog.golang.org/playground#Faking time) clockwork is inspired by @wickman's [threaded fake clock](https://gist.github.com/wickman/3840816), and the [Golang playground](https://blog.golang.org/playground#TOC_3.1.)
## License
Apache License, Version 2.0. Please see [License File](LICENSE) for more information.
...@@ -11,6 +11,8 @@ type Clock interface { ...@@ -11,6 +11,8 @@ type Clock interface {
After(d time.Duration) <-chan time.Time After(d time.Duration) <-chan time.Time
Sleep(d time.Duration) Sleep(d time.Duration)
Now() time.Time Now() time.Time
Since(t time.Time) time.Duration
NewTicker(d time.Duration) Ticker
} }
// FakeClock provides an interface for a clock which can be // FakeClock provides an interface for a clock which can be
...@@ -60,6 +62,14 @@ func (rc *realClock) Now() time.Time { ...@@ -60,6 +62,14 @@ func (rc *realClock) Now() time.Time {
return time.Now() return time.Now()
} }
func (rc *realClock) Since(t time.Time) time.Duration {
return rc.Now().Sub(t)
}
func (rc *realClock) NewTicker(d time.Duration) Ticker {
return &realTicker{time.NewTicker(d)}
}
type fakeClock struct { type fakeClock struct {
sleepers []*sleeper sleepers []*sleeper
blockers []*blocker blockers []*blocker
...@@ -87,7 +97,7 @@ func (fc *fakeClock) After(d time.Duration) <-chan time.Time { ...@@ -87,7 +97,7 @@ func (fc *fakeClock) After(d time.Duration) <-chan time.Time {
defer fc.l.Unlock() defer fc.l.Unlock()
now := fc.time now := fc.time
done := make(chan time.Time, 1) done := make(chan time.Time, 1)
if d.Nanoseconds() == 0 { if d.Nanoseconds() <= 0 {
// special case - trigger immediately // special case - trigger immediately
done <- now done <- now
} else { } else {
...@@ -130,6 +140,22 @@ func (fc *fakeClock) Now() time.Time { ...@@ -130,6 +140,22 @@ func (fc *fakeClock) Now() time.Time {
return t return t
} }
// Since returns the duration that has passed since the given time on the fakeClock
func (fc *fakeClock) Since(t time.Time) time.Duration {
return fc.Now().Sub(t)
}
func (fc *fakeClock) NewTicker(d time.Duration) Ticker {
ft := &fakeTicker{
c: make(chan time.Time, 1),
stop: make(chan bool, 1),
clock: fc,
period: d,
}
ft.runTickThread()
return ft
}
// Advance advances fakeClock to a new point in time, ensuring channels from any // Advance advances fakeClock to a new point in time, ensuring channels from any
// previous invocations of After are notified appropriately before returning // previous invocations of After are notified appropriately before returning
func (fc *fakeClock) Advance(d time.Duration) { func (fc *fakeClock) Advance(d time.Duration) {
......
module github.com/jonboulle/clockwork
go 1.13
package clockwork
import (
"time"
)
// Ticker provides an interface which can be used instead of directly
// using the ticker within the time module. The real-time ticker t
// provides ticks through t.C which becomes now t.Chan() to make
// this channel requirement definable in this interface.
type Ticker interface {
Chan() <-chan time.Time
Stop()
}
type realTicker struct{ *time.Ticker }
func (rt *realTicker) Chan() <-chan time.Time {
return rt.C
}
type fakeTicker struct {
c chan time.Time
stop chan bool
clock FakeClock
period time.Duration
}
func (ft *fakeTicker) Chan() <-chan time.Time {
return ft.c
}
func (ft *fakeTicker) Stop() {
ft.stop <- true
}
// runTickThread initializes a background goroutine to send the tick time to the ticker channel
// after every period. Tick events are discarded if the underlying ticker channel does not have
// enough capacity.
func (ft *fakeTicker) runTickThread() {
nextTick := ft.clock.Now().Add(ft.period)
next := ft.clock.After(ft.period)
go func() {
for {
select {
case <-ft.stop:
return
case <-next:
// We send the time that the tick was supposed to occur at.
tick := nextTick
// Before sending the tick, we'll compute the next tick time and star the clock.After call.
now := ft.clock.Now()
// First, figure out how many periods there have been between "now" and the time we were
// supposed to have trigged, then advance over all of those.
skipTicks := (now.Sub(tick) + ft.period - 1) / ft.period
nextTick = nextTick.Add(skipTicks * ft.period)
// Now, keep advancing until we are past now. This should happen at most once.
for !nextTick.After(now) {
nextTick = nextTick.Add(ft.period)
}
// Figure out how long between now and the next scheduled tick, then wait that long.
remaining := nextTick.Sub(now)
next = ft.clock.After(remaining)
// Finally, we can actually send the tick.
select {
case ft.c <- tick:
default:
}
}
}
}()
}
...@@ -9,6 +9,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD ...@@ -9,6 +9,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
...@@ -288,6 +288,9 @@ non_decimal_loop: ...@@ -288,6 +288,9 @@ non_decimal_loop:
return iter.readFloat64SlowPath() return iter.readFloat64SlowPath()
} }
value = (value << 3) + (value << 1) + uint64(ind) value = (value << 3) + (value << 1) + uint64(ind)
if value > maxFloat64 {
return iter.readFloat64SlowPath()
}
} }
} }
return iter.readFloat64SlowPath() return iter.readFloat64SlowPath()
......
...@@ -9,6 +9,7 @@ var intDigits []int8 ...@@ -9,6 +9,7 @@ var intDigits []int8
const uint32SafeToMultiply10 = uint32(0xffffffff)/10 - 1 const uint32SafeToMultiply10 = uint32(0xffffffff)/10 - 1
const uint64SafeToMultiple10 = uint64(0xffffffffffffffff)/10 - 1 const uint64SafeToMultiple10 = uint64(0xffffffffffffffff)/10 - 1
const maxFloat64 = 1<<53 - 1
func init() { func init() {
intDigits = make([]int8, 256) intDigits = make([]int8, 256)
...@@ -339,7 +340,7 @@ func (iter *Iterator) readUint64(c byte) (ret uint64) { ...@@ -339,7 +340,7 @@ func (iter *Iterator) readUint64(c byte) (ret uint64) {
} }
func (iter *Iterator) assertInteger() { func (iter *Iterator) assertInteger() {
if iter.head < len(iter.buf) && iter.buf[iter.head] == '.' { if iter.head < iter.tail && iter.buf[iter.head] == '.' {
iter.ReportError("assertInteger", "can not decode float as int") iter.ReportError("assertInteger", "can not decode float as int")
} }
} }
...@@ -65,7 +65,7 @@ func (iter *Iterator) ReadVal(obj interface{}) { ...@@ -65,7 +65,7 @@ func (iter *Iterator) ReadVal(obj interface{}) {
decoder := iter.cfg.getDecoderFromCache(cacheKey) decoder := iter.cfg.getDecoderFromCache(cacheKey)
if decoder == nil { if decoder == nil {
typ := reflect2.TypeOf(obj) typ := reflect2.TypeOf(obj)
if typ.Kind() != reflect.Ptr { if typ == nil || typ.Kind() != reflect.Ptr {
iter.ReportError("ReadVal", "can only unmarshal into pointer") iter.ReportError("ReadVal", "can only unmarshal into pointer")
return return
} }
......
...@@ -33,11 +33,19 @@ type jsonRawMessageCodec struct { ...@@ -33,11 +33,19 @@ type jsonRawMessageCodec struct {
} }
func (codec *jsonRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { func (codec *jsonRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) {
*((*json.RawMessage)(ptr)) = json.RawMessage(iter.SkipAndReturnBytes()) if iter.ReadNil() {
*((*json.RawMessage)(ptr)) = nil
} else {
*((*json.RawMessage)(ptr)) = iter.SkipAndReturnBytes()
}
} }
func (codec *jsonRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) { func (codec *jsonRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) {
stream.WriteRaw(string(*((*json.RawMessage)(ptr)))) if *((*json.RawMessage)(ptr)) == nil {
stream.WriteNil()
} else {
stream.WriteRaw(string(*((*json.RawMessage)(ptr))))
}
} }
func (codec *jsonRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { func (codec *jsonRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool {
...@@ -48,11 +56,19 @@ type jsoniterRawMessageCodec struct { ...@@ -48,11 +56,19 @@ type jsoniterRawMessageCodec struct {
} }
func (codec *jsoniterRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { func (codec *jsoniterRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) {
*((*RawMessage)(ptr)) = RawMessage(iter.SkipAndReturnBytes()) if iter.ReadNil() {
*((*RawMessage)(ptr)) = nil
} else {
*((*RawMessage)(ptr)) = iter.SkipAndReturnBytes()
}
} }
func (codec *jsoniterRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) { func (codec *jsoniterRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) {
stream.WriteRaw(string(*((*RawMessage)(ptr)))) if *((*RawMessage)(ptr)) == nil {
stream.WriteNil()
} else {
stream.WriteRaw(string(*((*RawMessage)(ptr))))
}
} }
func (codec *jsoniterRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { func (codec *jsoniterRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool {
......
...@@ -1075,6 +1075,11 @@ type stringModeNumberDecoder struct { ...@@ -1075,6 +1075,11 @@ type stringModeNumberDecoder struct {
} }
func (decoder *stringModeNumberDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { func (decoder *stringModeNumberDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) {
if iter.WhatIsNext() == NilValue {
decoder.elemDecoder.Decode(ptr, iter)
return
}
c := iter.nextToken() c := iter.nextToken()
if c != '"' { if c != '"' {
iter.ReportError("stringModeNumberDecoder", `expect ", but found `+string([]byte{c})) iter.ReportError("stringModeNumberDecoder", `expect ", but found `+string([]byte{c}))
......
...@@ -163,7 +163,7 @@ func (c *counter) updateExemplar(v float64, l Labels) { ...@@ -163,7 +163,7 @@ func (c *counter) updateExemplar(v float64, l Labels) {
// (e.g. number of HTTP requests, partitioned by response code and // (e.g. number of HTTP requests, partitioned by response code and
// method). Create instances with NewCounterVec. // method). Create instances with NewCounterVec.
type CounterVec struct { type CounterVec struct {
*metricVec *MetricVec
} }
// NewCounterVec creates a new CounterVec based on the provided CounterOpts and // NewCounterVec creates a new CounterVec based on the provided CounterOpts and
...@@ -176,11 +176,11 @@ func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec { ...@@ -176,11 +176,11 @@ func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec {
opts.ConstLabels, opts.ConstLabels,
) )
return &CounterVec{ return &CounterVec{
metricVec: newMetricVec(desc, func(lvs ...string) Metric { MetricVec: NewMetricVec(desc, func(lvs ...string) Metric {
if len(lvs) != len(desc.variableLabels) { if len(lvs) != len(desc.variableLabels) {
panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, lvs)) panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, lvs))
} }
result := &counter{desc: desc, labelPairs: makeLabelPairs(desc, lvs), now: time.Now} result := &counter{desc: desc, labelPairs: MakeLabelPairs(desc, lvs), now: time.Now}
result.init(result) // Init self-collection. result.init(result) // Init self-collection.
return result return result
}), }),
...@@ -188,7 +188,7 @@ func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec { ...@@ -188,7 +188,7 @@ func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec {
} }
// GetMetricWithLabelValues returns the Counter for the given slice of label // GetMetricWithLabelValues returns the Counter for the given slice of label
// values (same order as the VariableLabels in Desc). If that combination of // values (same order as the variable labels in Desc). If that combination of
// label values is accessed for the first time, a new Counter is created. // label values is accessed for the first time, a new Counter is created.
// //
// It is possible to call this method without using the returned Counter to only // It is possible to call this method without using the returned Counter to only
...@@ -202,7 +202,7 @@ func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec { ...@@ -202,7 +202,7 @@ func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec {
// Counter with the same label values is created later. // Counter with the same label values is created later.
// //
// An error is returned if the number of label values is not the same as the // An error is returned if the number of label values is not the same as the
// number of VariableLabels in Desc (minus any curried labels). // number of variable labels in Desc (minus any curried labels).
// //
// Note that for more than one label value, this method is prone to mistakes // Note that for more than one label value, this method is prone to mistakes
// caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as // caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as
...@@ -211,7 +211,7 @@ func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec { ...@@ -211,7 +211,7 @@ func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec {
// with a performance overhead (for creating and processing the Labels map). // with a performance overhead (for creating and processing the Labels map).
// See also the GaugeVec example. // See also the GaugeVec example.
func (v *CounterVec) GetMetricWithLabelValues(lvs ...string) (Counter, error) { func (v *CounterVec) GetMetricWithLabelValues(lvs ...string) (Counter, error) {
metric, err := v.metricVec.getMetricWithLabelValues(lvs...) metric, err := v.MetricVec.GetMetricWithLabelValues(lvs...)
if metric != nil { if metric != nil {
return metric.(Counter), err return metric.(Counter), err
} }
...@@ -219,19 +219,19 @@ func (v *CounterVec) GetMetricWithLabelValues(lvs ...string) (Counter, error) { ...@@ -219,19 +219,19 @@ func (v *CounterVec) GetMetricWithLabelValues(lvs ...string) (Counter, error) {
} }
// GetMetricWith returns the Counter for the given Labels map (the label names // GetMetricWith returns the Counter for the given Labels map (the label names
// must match those of the VariableLabels in Desc). If that label map is // must match those of the variable labels in Desc). If that label map is
// accessed for the first time, a new Counter is created. Implications of // accessed for the first time, a new Counter is created. Implications of
// creating a Counter without using it and keeping the Counter for later use are // creating a Counter without using it and keeping the Counter for later use are
// the same as for GetMetricWithLabelValues. // the same as for GetMetricWithLabelValues.
// //
// An error is returned if the number and names of the Labels are inconsistent // An error is returned if the number and names of the Labels are inconsistent
// with those of the VariableLabels in Desc (minus any curried labels). // with those of the variable labels in Desc (minus any curried labels).
// //
// This method is used for the same purpose as // This method is used for the same purpose as
// GetMetricWithLabelValues(...string). See there for pros and cons of the two // GetMetricWithLabelValues(...string). See there for pros and cons of the two
// methods. // methods.
func (v *CounterVec) GetMetricWith(labels Labels) (Counter, error) { func (v *CounterVec) GetMetricWith(labels Labels) (Counter, error) {
metric, err := v.metricVec.getMetricWith(labels) metric, err := v.MetricVec.GetMetricWith(labels)
if metric != nil { if metric != nil {
return metric.(Counter), err return metric.(Counter), err
} }
...@@ -275,7 +275,7 @@ func (v *CounterVec) With(labels Labels) Counter { ...@@ -275,7 +275,7 @@ func (v *CounterVec) With(labels Labels) Counter {
// registered with a given registry (usually the uncurried version). The Reset // registered with a given registry (usually the uncurried version). The Reset
// method deletes all metrics, even if called on a curried vector. // method deletes all metrics, even if called on a curried vector.
func (v *CounterVec) CurryWith(labels Labels) (*CounterVec, error) { func (v *CounterVec) CurryWith(labels Labels) (*CounterVec, error) {
vec, err := v.curryWith(labels) vec, err := v.MetricVec.CurryWith(labels)
if vec != nil { if vec != nil {
return &CounterVec{vec}, err return &CounterVec{vec}, err
} }
......
...@@ -20,7 +20,7 @@ import ( ...@@ -20,7 +20,7 @@ import (
"strings" "strings"
"github.com/cespare/xxhash/v2" "github.com/cespare/xxhash/v2"
//lint:ignore SA1019 Need to keep deprecated package for compatibility. //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/prometheus/common/model" "github.com/prometheus/common/model"
...@@ -51,7 +51,7 @@ type Desc struct { ...@@ -51,7 +51,7 @@ type Desc struct {
// constLabelPairs contains precalculated DTO label pairs based on // constLabelPairs contains precalculated DTO label pairs based on
// the constant labels. // the constant labels.
constLabelPairs []*dto.LabelPair constLabelPairs []*dto.LabelPair
// VariableLabels contains names of labels for which the metric // variableLabels contains names of labels for which the metric
// maintains variable values. // maintains variable values.
variableLabels []string variableLabels []string
// id is a hash of the values of the ConstLabels and fqName. This // id is a hash of the values of the ConstLabels and fqName. This
......
...@@ -22,43 +22,10 @@ type expvarCollector struct { ...@@ -22,43 +22,10 @@ type expvarCollector struct {
exports map[string]*Desc exports map[string]*Desc
} }
// NewExpvarCollector returns a newly allocated expvar Collector that still has // NewExpvarCollector is the obsolete version of collectors.NewExpvarCollector.
// to be registered with a Prometheus registry. // See there for documentation.
// //
// An expvar Collector collects metrics from the expvar interface. It provides a // Deprecated: Use collectors.NewExpvarCollector instead.
// quick way to expose numeric values that are already exported via expvar as
// Prometheus metrics. Note that the data models of expvar and Prometheus are
// fundamentally different, and that the expvar Collector is inherently slower
// than native Prometheus metrics. Thus, the expvar Collector is probably great
// for experiments and prototying, but you should seriously consider a more
// direct implementation of Prometheus metrics for monitoring production
// systems.
//
// The exports map has the following meaning:
//
// The keys in the map correspond to expvar keys, i.e. for every expvar key you
// want to export as Prometheus metric, you need an entry in the exports
// map. The descriptor mapped to each key describes how to export the expvar
// value. It defines the name and the help string of the Prometheus metric
// proxying the expvar value. The type will always be Untyped.
//
// For descriptors without variable labels, the expvar value must be a number or
// a bool. The number is then directly exported as the Prometheus sample
// value. (For a bool, 'false' translates to 0 and 'true' to 1). Expvar values
// that are not numbers or bools are silently ignored.
//
// If the descriptor has one variable label, the expvar value must be an expvar
// map. The keys in the expvar map become the various values of the one
// Prometheus label. The values in the expvar map must be numbers or bools again
// as above.
//
// For descriptors with more than one variable label, the expvar must be a
// nested expvar map, i.e. where the values of the topmost map are maps again
// etc. until a depth is reached that corresponds to the number of labels. The
// leaves of that structure must be numbers or bools as above to serve as the
// sample values.
//
// Anything that does not fit into the scheme above is silently ignored.
func NewExpvarCollector(exports map[string]*Desc) Collector { func NewExpvarCollector(exports map[string]*Desc) Collector {
return &expvarCollector{ return &expvarCollector{
exports: exports, exports: exports,
......
...@@ -132,7 +132,7 @@ func (g *gauge) Write(out *dto.Metric) error { ...@@ -132,7 +132,7 @@ func (g *gauge) Write(out *dto.Metric) error {
// (e.g. number of operations queued, partitioned by user and operation // (e.g. number of operations queued, partitioned by user and operation
// type). Create instances with NewGaugeVec. // type). Create instances with NewGaugeVec.
type GaugeVec struct { type GaugeVec struct {
*metricVec *MetricVec
} }
// NewGaugeVec creates a new GaugeVec based on the provided GaugeOpts and // NewGaugeVec creates a new GaugeVec based on the provided GaugeOpts and
...@@ -145,11 +145,11 @@ func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec { ...@@ -145,11 +145,11 @@ func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec {
opts.ConstLabels, opts.ConstLabels,
) )
return &GaugeVec{ return &GaugeVec{
metricVec: newMetricVec(desc, func(lvs ...string) Metric { MetricVec: NewMetricVec(desc, func(lvs ...string) Metric {
if len(lvs) != len(desc.variableLabels) { if len(lvs) != len(desc.variableLabels) {
panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, lvs)) panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, lvs))
} }
result := &gauge{desc: desc, labelPairs: makeLabelPairs(desc, lvs)} result := &gauge{desc: desc, labelPairs: MakeLabelPairs(desc, lvs)}
result.init(result) // Init self-collection. result.init(result) // Init self-collection.
return result return result
}), }),
...@@ -157,7 +157,7 @@ func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec { ...@@ -157,7 +157,7 @@ func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec {
} }
// GetMetricWithLabelValues returns the Gauge for the given slice of label // GetMetricWithLabelValues returns the Gauge for the given slice of label
// values (same order as the VariableLabels in Desc). If that combination of // values (same order as the variable labels in Desc). If that combination of
// label values is accessed for the first time, a new Gauge is created. // label values is accessed for the first time, a new Gauge is created.
// //
// It is possible to call this method without using the returned Gauge to only // It is possible to call this method without using the returned Gauge to only
...@@ -172,7 +172,7 @@ func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec { ...@@ -172,7 +172,7 @@ func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec {
// example. // example.
// //
// An error is returned if the number of label values is not the same as the // An error is returned if the number of label values is not the same as the
// number of VariableLabels in Desc (minus any curried labels). // number of variable labels in Desc (minus any curried labels).
// //
// Note that for more than one label value, this method is prone to mistakes // Note that for more than one label value, this method is prone to mistakes
// caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as // caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as
...@@ -180,7 +180,7 @@ func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec { ...@@ -180,7 +180,7 @@ func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec {
// latter has a much more readable (albeit more verbose) syntax, but it comes // latter has a much more readable (albeit more verbose) syntax, but it comes
// with a performance overhead (for creating and processing the Labels map). // with a performance overhead (for creating and processing the Labels map).
func (v *GaugeVec) GetMetricWithLabelValues(lvs ...string) (Gauge, error) { func (v *GaugeVec) GetMetricWithLabelValues(lvs ...string) (Gauge, error) {
metric, err := v.metricVec.getMetricWithLabelValues(lvs...) metric, err := v.MetricVec.GetMetricWithLabelValues(lvs...)
if metric != nil { if metric != nil {
return metric.(Gauge), err return metric.(Gauge), err
} }
...@@ -188,19 +188,19 @@ func (v *GaugeVec) GetMetricWithLabelValues(lvs ...string) (Gauge, error) { ...@@ -188,19 +188,19 @@ func (v *GaugeVec) GetMetricWithLabelValues(lvs ...string) (Gauge, error) {
} }
// GetMetricWith returns the Gauge for the given Labels map (the label names // GetMetricWith returns the Gauge for the given Labels map (the label names
// must match those of the VariableLabels in Desc). If that label map is // must match those of the variable labels in Desc). If that label map is
// accessed for the first time, a new Gauge is created. Implications of // accessed for the first time, a new Gauge is created. Implications of
// creating a Gauge without using it and keeping the Gauge for later use are // creating a Gauge without using it and keeping the Gauge for later use are
// the same as for GetMetricWithLabelValues. // the same as for GetMetricWithLabelValues.
// //
// An error is returned if the number and names of the Labels are inconsistent // An error is returned if the number and names of the Labels are inconsistent
// with those of the VariableLabels in Desc (minus any curried labels). // with those of the variable labels in Desc (minus any curried labels).
// //
// This method is used for the same purpose as // This method is used for the same purpose as
// GetMetricWithLabelValues(...string). See there for pros and cons of the two // GetMetricWithLabelValues(...string). See there for pros and cons of the two
// methods. // methods.
func (v *GaugeVec) GetMetricWith(labels Labels) (Gauge, error) { func (v *GaugeVec) GetMetricWith(labels Labels) (Gauge, error) {
metric, err := v.metricVec.getMetricWith(labels) metric, err := v.MetricVec.GetMetricWith(labels)
if metric != nil { if metric != nil {
return metric.(Gauge), err return metric.(Gauge), err
} }
...@@ -244,7 +244,7 @@ func (v *GaugeVec) With(labels Labels) Gauge { ...@@ -244,7 +244,7 @@ func (v *GaugeVec) With(labels Labels) Gauge {
// registered with a given registry (usually the uncurried version). The Reset // registered with a given registry (usually the uncurried version). The Reset
// method deletes all metrics, even if called on a curried vector. // method deletes all metrics, even if called on a curried vector.
func (v *GaugeVec) CurryWith(labels Labels) (*GaugeVec, error) { func (v *GaugeVec) CurryWith(labels Labels) (*GaugeVec, error) {
vec, err := v.curryWith(labels) vec, err := v.MetricVec.CurryWith(labels)
if vec != nil { if vec != nil {
return &GaugeVec{vec}, err return &GaugeVec{vec}, err
} }
......
...@@ -36,31 +36,10 @@ type goCollector struct { ...@@ -36,31 +36,10 @@ type goCollector struct {
msMaxAge time.Duration // Maximum allowed age of old memstats. msMaxAge time.Duration // Maximum allowed age of old memstats.
} }
// NewGoCollector returns a collector that exports metrics about the current Go // NewGoCollector is the obsolete version of collectors.NewGoCollector.
// process. This includes memory stats. To collect those, runtime.ReadMemStats // See there for documentation.
// is called. This requires to “stop the world”, which usually only happens for
// garbage collection (GC). Take the following implications into account when
// deciding whether to use the Go collector:
// //
// 1. The performance impact of stopping the world is the more relevant the more // Deprecated: Use collectors.NewGoCollector instead.
// frequently metrics are collected. However, with Go1.9 or later the
// stop-the-world time per metrics collection is very short (~25µs) so that the
// performance impact will only matter in rare cases. However, with older Go
// versions, the stop-the-world duration depends on the heap size and can be
// quite significant (~1.7 ms/GiB as per
// https://go-review.googlesource.com/c/go/+/34937).
//
// 2. During an ongoing GC, nothing else can stop the world. Therefore, if the
// metrics collection happens to coincide with GC, it will only complete after
// GC has finished. Usually, GC is fast enough to not cause problems. However,
// with a very large heap, GC might take multiple seconds, which is enough to
// cause scrape timeouts in common setups. To avoid this problem, the Go
// collector will use the memstats from a previous collection if
// runtime.ReadMemStats takes more than 1s. However, if there are no previously
// collected memstats, or their collection is more than 5m ago, the collection
// will block until runtime.ReadMemStats succeeds. (The problem might be solved
// in Go1.13, see https://github.com/golang/go/issues/19812 for the related Go
// issue.)
func NewGoCollector() Collector { func NewGoCollector() Collector {
return &goCollector{ return &goCollector{
goroutinesDesc: NewDesc( goroutinesDesc: NewDesc(
...@@ -365,25 +344,17 @@ type memStatsMetrics []struct { ...@@ -365,25 +344,17 @@ type memStatsMetrics []struct {
valType ValueType valType ValueType
} }
// NewBuildInfoCollector returns a collector collecting a single metric // NewBuildInfoCollector is the obsolete version of collectors.NewBuildInfoCollector.
// "go_build_info" with the constant value 1 and three labels "path", "version", // See there for documentation.
// and "checksum". Their label values contain the main module path, version, and
// checksum, respectively. The labels will only have meaningful values if the
// binary is built with Go module support and from source code retrieved from
// the source repository (rather than the local file system). This is usually
// accomplished by building from outside of GOPATH, specifying the full address
// of the main package, e.g. "GO111MODULE=on go run
// github.com/prometheus/client_golang/examples/random". If built without Go
// module support, all label values will be "unknown". If built with Go module
// support but using the source code from the local file system, the "path" will
// be set appropriately, but "checksum" will be empty and "version" will be
// "(devel)".
// //
// This collector uses only the build information for the main module. See // Deprecated: Use collectors.NewBuildInfoCollector instead.
// https://github.com/povilasv/prommod for an example of a collector for the
// module dependencies.
func NewBuildInfoCollector() Collector { func NewBuildInfoCollector() Collector {
path, version, sum := readBuildInfo() path, version, sum := "unknown", "unknown", "unknown"
if bi, ok := debug.ReadBuildInfo(); ok {
path = bi.Main.Path
version = bi.Main.Version
sum = bi.Main.Sum
}
c := &selfCollector{MustNewConstMetric( c := &selfCollector{MustNewConstMetric(
NewDesc( NewDesc(
"go_build_info", "go_build_info",
......
...@@ -22,7 +22,7 @@ import ( ...@@ -22,7 +22,7 @@ import (
"sync/atomic" "sync/atomic"
"time" "time"
//lint:ignore SA1019 Need to keep deprecated package for compatibility. //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
dto "github.com/prometheus/client_model/go" dto "github.com/prometheus/client_model/go"
...@@ -47,7 +47,12 @@ type Histogram interface { ...@@ -47,7 +47,12 @@ type Histogram interface {
Metric Metric
Collector Collector
// Observe adds a single observation to the histogram. // Observe adds a single observation to the histogram. Observations are
// usually positive or zero. Negative observations are accepted but
// prevent current versions of Prometheus from properly detecting
// counter resets in the sum of observations. See
// https://prometheus.io/docs/practices/histograms/#count-and-sum-of-observations
// for details.
Observe(float64) Observe(float64)
} }
...@@ -192,7 +197,7 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr ...@@ -192,7 +197,7 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
h := &histogram{ h := &histogram{
desc: desc, desc: desc,
upperBounds: opts.Buckets, upperBounds: opts.Buckets,
labelPairs: makeLabelPairs(desc, labelValues), labelPairs: MakeLabelPairs(desc, labelValues),
counts: [2]*histogramCounts{{}, {}}, counts: [2]*histogramCounts{{}, {}},
now: time.Now, now: time.Now,
} }
...@@ -409,7 +414,7 @@ func (h *histogram) updateExemplar(v float64, bucket int, l Labels) { ...@@ -409,7 +414,7 @@ func (h *histogram) updateExemplar(v float64, bucket int, l Labels) {
// (e.g. HTTP request latencies, partitioned by status code and method). Create // (e.g. HTTP request latencies, partitioned by status code and method). Create
// instances with NewHistogramVec. // instances with NewHistogramVec.
type HistogramVec struct { type HistogramVec struct {
*metricVec *MetricVec
} }
// NewHistogramVec creates a new HistogramVec based on the provided HistogramOpts and // NewHistogramVec creates a new HistogramVec based on the provided HistogramOpts and
...@@ -422,14 +427,14 @@ func NewHistogramVec(opts HistogramOpts, labelNames []string) *HistogramVec { ...@@ -422,14 +427,14 @@ func NewHistogramVec(opts HistogramOpts, labelNames []string) *HistogramVec {
opts.ConstLabels, opts.ConstLabels,
) )
return &HistogramVec{ return &HistogramVec{
metricVec: newMetricVec(desc, func(lvs ...string) Metric { MetricVec: NewMetricVec(desc, func(lvs ...string) Metric {
return newHistogram(desc, opts, lvs...) return newHistogram(desc, opts, lvs...)
}), }),
} }
} }
// GetMetricWithLabelValues returns the Histogram for the given slice of label // GetMetricWithLabelValues returns the Histogram for the given slice of label
// values (same order as the VariableLabels in Desc). If that combination of // values (same order as the variable labels in Desc). If that combination of
// label values is accessed for the first time, a new Histogram is created. // label values is accessed for the first time, a new Histogram is created.
// //
// It is possible to call this method without using the returned Histogram to only // It is possible to call this method without using the returned Histogram to only
...@@ -444,7 +449,7 @@ func NewHistogramVec(opts HistogramOpts, labelNames []string) *HistogramVec { ...@@ -444,7 +449,7 @@ func NewHistogramVec(opts HistogramOpts, labelNames []string) *HistogramVec {
// example. // example.
// //
// An error is returned if the number of label values is not the same as the // An error is returned if the number of label values is not the same as the
// number of VariableLabels in Desc (minus any curried labels). // number of variable labels in Desc (minus any curried labels).
// //
// Note that for more than one label value, this method is prone to mistakes // Note that for more than one label value, this method is prone to mistakes
// caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as // caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as
...@@ -453,7 +458,7 @@ func NewHistogramVec(opts HistogramOpts, labelNames []string) *HistogramVec { ...@@ -453,7 +458,7 @@ func NewHistogramVec(opts HistogramOpts, labelNames []string) *HistogramVec {
// with a performance overhead (for creating and processing the Labels map). // with a performance overhead (for creating and processing the Labels map).
// See also the GaugeVec example. // See also the GaugeVec example.
func (v *HistogramVec) GetMetricWithLabelValues(lvs ...string) (Observer, error) { func (v *HistogramVec) GetMetricWithLabelValues(lvs ...string) (Observer, error) {
metric, err := v.metricVec.getMetricWithLabelValues(lvs...) metric, err := v.MetricVec.GetMetricWithLabelValues(lvs...)
if metric != nil { if metric != nil {
return metric.(Observer), err return metric.(Observer), err
} }
...@@ -461,19 +466,19 @@ func (v *HistogramVec) GetMetricWithLabelValues(lvs ...string) (Observer, error) ...@@ -461,19 +466,19 @@ func (v *HistogramVec) GetMetricWithLabelValues(lvs ...string) (Observer, error)
} }
// GetMetricWith returns the Histogram for the given Labels map (the label names // GetMetricWith returns the Histogram for the given Labels map (the label names
// must match those of the VariableLabels in Desc). If that label map is // must match those of the variable labels in Desc). If that label map is
// accessed for the first time, a new Histogram is created. Implications of // accessed for the first time, a new Histogram is created. Implications of
// creating a Histogram without using it and keeping the Histogram for later use // creating a Histogram without using it and keeping the Histogram for later use
// are the same as for GetMetricWithLabelValues. // are the same as for GetMetricWithLabelValues.
// //
// An error is returned if the number and names of the Labels are inconsistent // An error is returned if the number and names of the Labels are inconsistent
// with those of the VariableLabels in Desc (minus any curried labels). // with those of the variable labels in Desc (minus any curried labels).
// //
// This method is used for the same purpose as // This method is used for the same purpose as
// GetMetricWithLabelValues(...string). See there for pros and cons of the two // GetMetricWithLabelValues(...string). See there for pros and cons of the two
// methods. // methods.
func (v *HistogramVec) GetMetricWith(labels Labels) (Observer, error) { func (v *HistogramVec) GetMetricWith(labels Labels) (Observer, error) {
metric, err := v.metricVec.getMetricWith(labels) metric, err := v.MetricVec.GetMetricWith(labels)
if metric != nil { if metric != nil {
return metric.(Observer), err return metric.(Observer), err
} }
...@@ -517,7 +522,7 @@ func (v *HistogramVec) With(labels Labels) Observer { ...@@ -517,7 +522,7 @@ func (v *HistogramVec) With(labels Labels) Observer {
// registered with a given registry (usually the uncurried version). The Reset // registered with a given registry (usually the uncurried version). The Reset
// method deletes all metrics, even if called on a curried vector. // method deletes all metrics, even if called on a curried vector.
func (v *HistogramVec) CurryWith(labels Labels) (ObserverVec, error) { func (v *HistogramVec) CurryWith(labels Labels) (ObserverVec, error) {
vec, err := v.curryWith(labels) vec, err := v.MetricVec.CurryWith(labels)
if vec != nil { if vec != nil {
return &HistogramVec{vec}, err return &HistogramVec{vec}, err
} }
...@@ -602,7 +607,7 @@ func NewConstHistogram( ...@@ -602,7 +607,7 @@ func NewConstHistogram(
count: count, count: count,
sum: sum, sum: sum,
buckets: buckets, buckets: buckets,
labelPairs: makeLabelPairs(desc, labelValues), labelPairs: MakeLabelPairs(desc, labelValues),
}, nil }, nil
} }
......
...@@ -17,7 +17,7 @@ import ( ...@@ -17,7 +17,7 @@ import (
"strings" "strings"
"time" "time"
//lint:ignore SA1019 Need to keep deprecated package for compatibility. //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/prometheus/common/model" "github.com/prometheus/common/model"
...@@ -58,7 +58,7 @@ type Metric interface { ...@@ -58,7 +58,7 @@ type Metric interface {
} }
// Opts bundles the options for creating most Metric types. Each metric // Opts bundles the options for creating most Metric types. Each metric
// implementation XXX has its own XXXOpts type, but in most cases, it is just be // implementation XXX has its own XXXOpts type, but in most cases, it is just
// an alias of this type (which might change when the requirement arises.) // an alias of this type (which might change when the requirement arises.)
// //
// It is mandatory to set Name to a non-empty string. All other fields are // It is mandatory to set Name to a non-empty string. All other fields are
...@@ -89,7 +89,7 @@ type Opts struct { ...@@ -89,7 +89,7 @@ type Opts struct {
// better covered by target labels set by the scraping Prometheus // better covered by target labels set by the scraping Prometheus
// server, or by one specific metric (e.g. a build_info or a // server, or by one specific metric (e.g. a build_info or a
// machine_role metric). See also // machine_role metric). See also
// https://prometheus.io/docs/instrumenting/writing_exporters/#target-labels,-not-static-scraped-labels // https://prometheus.io/docs/instrumenting/writing_exporters/#target-labels-not-static-scraped-labels
ConstLabels Labels ConstLabels Labels
} }
......
...@@ -15,7 +15,11 @@ package prometheus ...@@ -15,7 +15,11 @@ package prometheus
import ( import (
"errors" "errors"
"fmt"
"io/ioutil"
"os" "os"
"strconv"
"strings"
) )
type processCollector struct { type processCollector struct {
...@@ -50,16 +54,10 @@ type ProcessCollectorOpts struct { ...@@ -50,16 +54,10 @@ type ProcessCollectorOpts struct {
ReportErrors bool ReportErrors bool
} }
// NewProcessCollector returns a collector which exports the current state of // NewProcessCollector is the obsolete version of collectors.NewProcessCollector.
// process metrics including CPU, memory and file descriptor usage as well as // See there for documentation.
// the process start time. The detailed behavior is defined by the provided
// ProcessCollectorOpts. The zero value of ProcessCollectorOpts creates a
// collector for the current process with an empty namespace string and no error
// reporting.
// //
// The collector only works on operating systems with a Linux-style proc // Deprecated: Use collectors.NewProcessCollector instead.
// filesystem and on Microsoft Windows. On other operating systems, it will not
// collect any metrics.
func NewProcessCollector(opts ProcessCollectorOpts) Collector { func NewProcessCollector(opts ProcessCollectorOpts) Collector {
ns := "" ns := ""
if len(opts.Namespace) > 0 { if len(opts.Namespace) > 0 {
...@@ -149,3 +147,20 @@ func (c *processCollector) reportError(ch chan<- Metric, desc *Desc, err error) ...@@ -149,3 +147,20 @@ func (c *processCollector) reportError(ch chan<- Metric, desc *Desc, err error)
} }
ch <- NewInvalidMetric(desc, err) ch <- NewInvalidMetric(desc, err)
} }
// NewPidFileFn returns a function that retrieves a pid from the specified file.
// It is meant to be used for the PidFn field in ProcessCollectorOpts.
func NewPidFileFn(pidFilePath string) func() (int, error) {
return func() (int, error) {
content, err := ioutil.ReadFile(pidFilePath)
if err != nil {
return 0, fmt.Errorf("can't read pid file %q: %+v", pidFilePath, err)
}
pid, err := strconv.Atoi(strings.TrimSpace(string(content)))
if err != nil {
return 0, fmt.Errorf("can't parse pid file %q: %+v", pidFilePath, err)
}
return pid, nil
}
}
...@@ -83,8 +83,7 @@ type readerFromDelegator struct{ *responseWriterDelegator } ...@@ -83,8 +83,7 @@ type readerFromDelegator struct{ *responseWriterDelegator }
type pusherDelegator struct{ *responseWriterDelegator } type pusherDelegator struct{ *responseWriterDelegator }
func (d closeNotifierDelegator) CloseNotify() <-chan bool { func (d closeNotifierDelegator) CloseNotify() <-chan bool {
//lint:ignore SA1019 http.CloseNotifier is deprecated but we don't want to //nolint:staticcheck // Ignore SA1019. http.CloseNotifier is deprecated but we keep it here to not break existing users.
//remove support from client_golang yet.
return d.ResponseWriter.(http.CloseNotifier).CloseNotify() return d.ResponseWriter.(http.CloseNotifier).CloseNotify()
} }
func (d flusherDelegator) Flush() { func (d flusherDelegator) Flush() {
...@@ -348,8 +347,7 @@ func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) deleg ...@@ -348,8 +347,7 @@ func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) deleg
} }
id := 0 id := 0
//lint:ignore SA1019 http.CloseNotifier is deprecated but we don't want to //nolint:staticcheck // Ignore SA1019. http.CloseNotifier is deprecated but we keep it here to not break existing users.
//remove support from client_golang yet.
if _, ok := w.(http.CloseNotifier); ok { if _, ok := w.(http.CloseNotifier); ok {
id += closeNotifier id += closeNotifier
} }
......
...@@ -99,7 +99,7 @@ func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler { ...@@ -99,7 +99,7 @@ func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler {
inFlightSem = make(chan struct{}, opts.MaxRequestsInFlight) inFlightSem = make(chan struct{}, opts.MaxRequestsInFlight)
} }
if opts.Registry != nil { if opts.Registry != nil {
// Initialize all possibilites that can occur below. // Initialize all possibilities that can occur below.
errCnt.WithLabelValues("gathering") errCnt.WithLabelValues("gathering")
errCnt.WithLabelValues("encoding") errCnt.WithLabelValues("encoding")
if err := opts.Registry.Register(errCnt); err != nil { if err := opts.Registry.Register(errCnt); err != nil {
...@@ -303,8 +303,12 @@ type Logger interface { ...@@ -303,8 +303,12 @@ type Logger interface {
// HandlerOpts specifies options how to serve metrics via an http.Handler. The // HandlerOpts specifies options how to serve metrics via an http.Handler. The
// zero value of HandlerOpts is a reasonable default. // zero value of HandlerOpts is a reasonable default.
type HandlerOpts struct { type HandlerOpts struct {
// ErrorLog specifies an optional logger for errors collecting and // ErrorLog specifies an optional Logger for errors collecting and
// serving metrics. If nil, errors are not logged at all. // serving metrics. If nil, errors are not logged at all. Note that the
// type of a reported error is often prometheus.MultiError, which
// formats into a multi-line error string. If you want to avoid the
// latter, create a Logger implementation that detects a
// prometheus.MultiError and formats the contained errors into one line.
ErrorLog Logger ErrorLog Logger
// ErrorHandling defines how errors are handled. Note that errors are // ErrorHandling defines how errors are handled. Note that errors are
// logged regardless of the configured ErrorHandling provided ErrorLog // logged regardless of the configured ErrorHandling provided ErrorLog
......
...@@ -43,14 +43,14 @@ func InstrumentHandlerInFlight(g prometheus.Gauge, next http.Handler) http.Handl ...@@ -43,14 +43,14 @@ func InstrumentHandlerInFlight(g prometheus.Gauge, next http.Handler) http.Handl
// InstrumentHandlerDuration is a middleware that wraps the provided // InstrumentHandlerDuration is a middleware that wraps the provided
// http.Handler to observe the request duration with the provided ObserverVec. // http.Handler to observe the request duration with the provided ObserverVec.
// The ObserverVec must have zero, one, or two non-const non-curried labels. For // The ObserverVec must have valid metric and label names and must have zero,
// those, the only allowed label names are "code" and "method". The function // one, or two non-const non-curried labels. For those, the only allowed label
// panics otherwise. The Observe method of the Observer in the ObserverVec is // names are "code" and "method". The function panics otherwise. The Observe
// called with the request duration in seconds. Partitioning happens by HTTP // method of the Observer in the ObserverVec is called with the request duration
// status code and/or HTTP method if the respective instance label names are // in seconds. Partitioning happens by HTTP status code and/or HTTP method if
// present in the ObserverVec. For unpartitioned observations, use an // the respective instance label names are present in the ObserverVec. For
// ObserverVec with zero labels. Note that partitioning of Histograms is // unpartitioned observations, use an ObserverVec with zero labels. Note that
// expensive and should be used judiciously. // partitioning of Histograms is expensive and should be used judiciously.
// //
// If the wrapped Handler does not set a status code, a status code of 200 is assumed. // If the wrapped Handler does not set a status code, a status code of 200 is assumed.
// //
...@@ -79,12 +79,13 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler) ht ...@@ -79,12 +79,13 @@ func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler) ht
} }
// InstrumentHandlerCounter is a middleware that wraps the provided http.Handler // InstrumentHandlerCounter is a middleware that wraps the provided http.Handler
// to observe the request result with the provided CounterVec. The CounterVec // to observe the request result with the provided CounterVec. The CounterVec
// must have zero, one, or two non-const non-curried labels. For those, the only // must have valid metric and label names and must have zero, one, or two
// allowed label names are "code" and "method". The function panics // non-const non-curried labels. For those, the only allowed label names are
// otherwise. Partitioning of the CounterVec happens by HTTP status code and/or // "code" and "method". The function panics otherwise. Partitioning of the
// HTTP method if the respective instance label names are present in the // CounterVec happens by HTTP status code and/or HTTP method if the respective
// CounterVec. For unpartitioned counting, use a CounterVec with zero labels. // instance label names are present in the CounterVec. For unpartitioned
// counting, use a CounterVec with zero labels.
// //
// If the wrapped Handler does not set a status code, a status code of 200 is assumed. // If the wrapped Handler does not set a status code, a status code of 200 is assumed.
// //
...@@ -110,14 +111,15 @@ func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler) ...@@ -110,14 +111,15 @@ func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler)
// InstrumentHandlerTimeToWriteHeader is a middleware that wraps the provided // InstrumentHandlerTimeToWriteHeader is a middleware that wraps the provided
// http.Handler to observe with the provided ObserverVec the request duration // http.Handler to observe with the provided ObserverVec the request duration
// until the response headers are written. The ObserverVec must have zero, one, // until the response headers are written. The ObserverVec must have valid
// or two non-const non-curried labels. For those, the only allowed label names // metric and label names and must have zero, one, or two non-const non-curried
// are "code" and "method". The function panics otherwise. The Observe method of // labels. For those, the only allowed label names are "code" and "method". The
// the Observer in the ObserverVec is called with the request duration in // function panics otherwise. The Observe method of the Observer in the
// seconds. Partitioning happens by HTTP status code and/or HTTP method if the // ObserverVec is called with the request duration in seconds. Partitioning
// respective instance label names are present in the ObserverVec. For // happens by HTTP status code and/or HTTP method if the respective instance
// unpartitioned observations, use an ObserverVec with zero labels. Note that // label names are present in the ObserverVec. For unpartitioned observations,
// partitioning of Histograms is expensive and should be used judiciously. // use an ObserverVec with zero labels. Note that partitioning of Histograms is
// expensive and should be used judiciously.
// //
// If the wrapped Handler panics before calling WriteHeader, no value is // If the wrapped Handler panics before calling WriteHeader, no value is
// reported. // reported.
...@@ -139,15 +141,15 @@ func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Ha ...@@ -139,15 +141,15 @@ func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Ha
} }
// InstrumentHandlerRequestSize is a middleware that wraps the provided // InstrumentHandlerRequestSize is a middleware that wraps the provided
// http.Handler to observe the request size with the provided ObserverVec. The // http.Handler to observe the request size with the provided ObserverVec. The
// ObserverVec must have zero, one, or two non-const non-curried labels. For // ObserverVec must have valid metric and label names and must have zero, one,
// those, the only allowed label names are "code" and "method". The function // or two non-const non-curried labels. For those, the only allowed label names
// panics otherwise. The Observe method of the Observer in the ObserverVec is // are "code" and "method". The function panics otherwise. The Observe method of
// called with the request size in bytes. Partitioning happens by HTTP status // the Observer in the ObserverVec is called with the request size in
// code and/or HTTP method if the respective instance label names are present in // bytes. Partitioning happens by HTTP status code and/or HTTP method if the
// the ObserverVec. For unpartitioned observations, use an ObserverVec with zero // respective instance label names are present in the ObserverVec. For
// labels. Note that partitioning of Histograms is expensive and should be used // unpartitioned observations, use an ObserverVec with zero labels. Note that
// judiciously. // partitioning of Histograms is expensive and should be used judiciously.
// //
// If the wrapped Handler does not set a status code, a status code of 200 is assumed. // If the wrapped Handler does not set a status code, a status code of 200 is assumed.
// //
...@@ -174,15 +176,15 @@ func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler) ...@@ -174,15 +176,15 @@ func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler)
} }
// InstrumentHandlerResponseSize is a middleware that wraps the provided // InstrumentHandlerResponseSize is a middleware that wraps the provided
// http.Handler to observe the response size with the provided ObserverVec. The // http.Handler to observe the response size with the provided ObserverVec. The
// ObserverVec must have zero, one, or two non-const non-curried labels. For // ObserverVec must have valid metric and label names and must have zero, one,
// those, the only allowed label names are "code" and "method". The function // or two non-const non-curried labels. For those, the only allowed label names
// panics otherwise. The Observe method of the Observer in the ObserverVec is // are "code" and "method". The function panics otherwise. The Observe method of
// called with the response size in bytes. Partitioning happens by HTTP status // the Observer in the ObserverVec is called with the response size in
// code and/or HTTP method if the respective instance label names are present in // bytes. Partitioning happens by HTTP status code and/or HTTP method if the
// the ObserverVec. For unpartitioned observations, use an ObserverVec with zero // respective instance label names are present in the ObserverVec. For
// labels. Note that partitioning of Histograms is expensive and should be used // unpartitioned observations, use an ObserverVec with zero labels. Note that
// judiciously. // partitioning of Histograms is expensive and should be used judiciously.
// //
// If the wrapped Handler does not set a status code, a status code of 200 is assumed. // If the wrapped Handler does not set a status code, a status code of 200 is assumed.
// //
...@@ -198,6 +200,11 @@ func InstrumentHandlerResponseSize(obs prometheus.ObserverVec, next http.Handler ...@@ -198,6 +200,11 @@ func InstrumentHandlerResponseSize(obs prometheus.ObserverVec, next http.Handler
}) })
} }
// checkLabels returns whether the provided Collector has a non-const,
// non-curried label named "code" and/or "method". It panics if the provided
// Collector does not have a Desc or has more than one Desc or its Desc is
// invalid. It also panics if the Collector has any non-const, non-curried
// labels that are not named "code" or "method".
func checkLabels(c prometheus.Collector) (code bool, method bool) { func checkLabels(c prometheus.Collector) (code bool, method bool) {
// TODO(beorn7): Remove this hacky way to check for instance labels // TODO(beorn7): Remove this hacky way to check for instance labels
// once Descriptors can have their dimensionality queried. // once Descriptors can have their dimensionality queried.
...@@ -225,6 +232,10 @@ func checkLabels(c prometheus.Collector) (code bool, method bool) { ...@@ -225,6 +232,10 @@ func checkLabels(c prometheus.Collector) (code bool, method bool) {
close(descc) close(descc)
// Make sure the Collector has a valid Desc by registering it with a
// temporary registry.
prometheus.NewRegistry().MustRegister(c)
// Create a ConstMetric with the Desc. Since we don't know how many // Create a ConstMetric with the Desc. Since we don't know how many
// variable labels there are, try for as long as it needs. // variable labels there are, try for as long as it needs.
for err := errors.New("dummy"); err != nil; lvs = append(lvs, magicString) { for err := errors.New("dummy"); err != nil; lvs = append(lvs, magicString) {
......
...@@ -26,7 +26,7 @@ import ( ...@@ -26,7 +26,7 @@ import (
"unicode/utf8" "unicode/utf8"
"github.com/cespare/xxhash/v2" "github.com/cespare/xxhash/v2"
//lint:ignore SA1019 Need to keep deprecated package for compatibility. //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/prometheus/common/expfmt" "github.com/prometheus/common/expfmt"
...@@ -215,6 +215,8 @@ func (err AlreadyRegisteredError) Error() string { ...@@ -215,6 +215,8 @@ func (err AlreadyRegisteredError) Error() string {
// by a Gatherer to report multiple errors during MetricFamily gathering. // by a Gatherer to report multiple errors during MetricFamily gathering.
type MultiError []error type MultiError []error
// Error formats the contained errors as a bullet point list, preceded by the
// total number of errors. Note that this results in a multi-line string.
func (errs MultiError) Error() string { func (errs MultiError) Error() string {
if len(errs) == 0 { if len(errs) == 0 {
return "" return ""
......
...@@ -23,7 +23,7 @@ import ( ...@@ -23,7 +23,7 @@ import (
"time" "time"
"github.com/beorn7/perks/quantile" "github.com/beorn7/perks/quantile"
//lint:ignore SA1019 Need to keep deprecated package for compatibility. //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
dto "github.com/prometheus/client_model/go" dto "github.com/prometheus/client_model/go"
...@@ -55,7 +55,12 @@ type Summary interface { ...@@ -55,7 +55,12 @@ type Summary interface {
Metric Metric
Collector Collector
// Observe adds a single observation to the summary. // Observe adds a single observation to the summary. Observations are
// usually positive or zero. Negative observations are accepted but
// prevent current versions of Prometheus from properly detecting
// counter resets in the sum of observations. See
// https://prometheus.io/docs/practices/histograms/#count-and-sum-of-observations
// for details.
Observe(float64) Observe(float64)
} }
...@@ -110,7 +115,7 @@ type SummaryOpts struct { ...@@ -110,7 +115,7 @@ type SummaryOpts struct {
// better covered by target labels set by the scraping Prometheus // better covered by target labels set by the scraping Prometheus
// server, or by one specific metric (e.g. a build_info or a // server, or by one specific metric (e.g. a build_info or a
// machine_role metric). See also // machine_role metric). See also
// https://prometheus.io/docs/instrumenting/writing_exporters/#target-labels,-not-static-scraped-labels // https://prometheus.io/docs/instrumenting/writing_exporters/#target-labels-not-static-scraped-labels
ConstLabels Labels ConstLabels Labels
// Objectives defines the quantile rank estimates with their respective // Objectives defines the quantile rank estimates with their respective
...@@ -121,7 +126,9 @@ type SummaryOpts struct { ...@@ -121,7 +126,9 @@ type SummaryOpts struct {
Objectives map[float64]float64 Objectives map[float64]float64
// MaxAge defines the duration for which an observation stays relevant // MaxAge defines the duration for which an observation stays relevant
// for the summary. Must be positive. The default value is DefMaxAge. // for the summary. Only applies to pre-calculated quantiles, does not
// apply to _sum and _count. Must be positive. The default value is
// DefMaxAge.
MaxAge time.Duration MaxAge time.Duration
// AgeBuckets is the number of buckets used to exclude observations that // AgeBuckets is the number of buckets used to exclude observations that
...@@ -208,7 +215,7 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary { ...@@ -208,7 +215,7 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
// Use the lock-free implementation of a Summary without objectives. // Use the lock-free implementation of a Summary without objectives.
s := &noObjectivesSummary{ s := &noObjectivesSummary{
desc: desc, desc: desc,
labelPairs: makeLabelPairs(desc, labelValues), labelPairs: MakeLabelPairs(desc, labelValues),
counts: [2]*summaryCounts{{}, {}}, counts: [2]*summaryCounts{{}, {}},
} }
s.init(s) // Init self-collection. s.init(s) // Init self-collection.
...@@ -221,7 +228,7 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary { ...@@ -221,7 +228,7 @@ func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
objectives: opts.Objectives, objectives: opts.Objectives,
sortedObjectives: make([]float64, 0, len(opts.Objectives)), sortedObjectives: make([]float64, 0, len(opts.Objectives)),
labelPairs: makeLabelPairs(desc, labelValues), labelPairs: MakeLabelPairs(desc, labelValues),
hotBuf: make([]float64, 0, opts.BufCap), hotBuf: make([]float64, 0, opts.BufCap),
coldBuf: make([]float64, 0, opts.BufCap), coldBuf: make([]float64, 0, opts.BufCap),
...@@ -513,7 +520,7 @@ func (s quantSort) Less(i, j int) bool { ...@@ -513,7 +520,7 @@ func (s quantSort) Less(i, j int) bool {
// (e.g. HTTP request latencies, partitioned by status code and method). Create // (e.g. HTTP request latencies, partitioned by status code and method). Create
// instances with NewSummaryVec. // instances with NewSummaryVec.
type SummaryVec struct { type SummaryVec struct {
*metricVec *MetricVec
} }
// NewSummaryVec creates a new SummaryVec based on the provided SummaryOpts and // NewSummaryVec creates a new SummaryVec based on the provided SummaryOpts and
...@@ -535,14 +542,14 @@ func NewSummaryVec(opts SummaryOpts, labelNames []string) *SummaryVec { ...@@ -535,14 +542,14 @@ func NewSummaryVec(opts SummaryOpts, labelNames []string) *SummaryVec {
opts.ConstLabels, opts.ConstLabels,
) )
return &SummaryVec{ return &SummaryVec{
metricVec: newMetricVec(desc, func(lvs ...string) Metric { MetricVec: NewMetricVec(desc, func(lvs ...string) Metric {
return newSummary(desc, opts, lvs...) return newSummary(desc, opts, lvs...)
}), }),
} }
} }
// GetMetricWithLabelValues returns the Summary for the given slice of label // GetMetricWithLabelValues returns the Summary for the given slice of label
// values (same order as the VariableLabels in Desc). If that combination of // values (same order as the variable labels in Desc). If that combination of
// label values is accessed for the first time, a new Summary is created. // label values is accessed for the first time, a new Summary is created.
// //
// It is possible to call this method without using the returned Summary to only // It is possible to call this method without using the returned Summary to only
...@@ -557,7 +564,7 @@ func NewSummaryVec(opts SummaryOpts, labelNames []string) *SummaryVec { ...@@ -557,7 +564,7 @@ func NewSummaryVec(opts SummaryOpts, labelNames []string) *SummaryVec {
// example. // example.
// //
// An error is returned if the number of label values is not the same as the // An error is returned if the number of label values is not the same as the
// number of VariableLabels in Desc (minus any curried labels). // number of variable labels in Desc (minus any curried labels).
// //
// Note that for more than one label value, this method is prone to mistakes // Note that for more than one label value, this method is prone to mistakes
// caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as // caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as
...@@ -566,7 +573,7 @@ func NewSummaryVec(opts SummaryOpts, labelNames []string) *SummaryVec { ...@@ -566,7 +573,7 @@ func NewSummaryVec(opts SummaryOpts, labelNames []string) *SummaryVec {
// with a performance overhead (for creating and processing the Labels map). // with a performance overhead (for creating and processing the Labels map).
// See also the GaugeVec example. // See also the GaugeVec example.
func (v *SummaryVec) GetMetricWithLabelValues(lvs ...string) (Observer, error) { func (v *SummaryVec) GetMetricWithLabelValues(lvs ...string) (Observer, error) {
metric, err := v.metricVec.getMetricWithLabelValues(lvs...) metric, err := v.MetricVec.GetMetricWithLabelValues(lvs...)
if metric != nil { if metric != nil {
return metric.(Observer), err return metric.(Observer), err
} }
...@@ -574,19 +581,19 @@ func (v *SummaryVec) GetMetricWithLabelValues(lvs ...string) (Observer, error) { ...@@ -574,19 +581,19 @@ func (v *SummaryVec) GetMetricWithLabelValues(lvs ...string) (Observer, error) {
} }
// GetMetricWith returns the Summary for the given Labels map (the label names // GetMetricWith returns the Summary for the given Labels map (the label names
// must match those of the VariableLabels in Desc). If that label map is // must match those of the variable labels in Desc). If that label map is
// accessed for the first time, a new Summary is created. Implications of // accessed for the first time, a new Summary is created. Implications of
// creating a Summary without using it and keeping the Summary for later use are // creating a Summary without using it and keeping the Summary for later use are
// the same as for GetMetricWithLabelValues. // the same as for GetMetricWithLabelValues.
// //
// An error is returned if the number and names of the Labels are inconsistent // An error is returned if the number and names of the Labels are inconsistent
// with those of the VariableLabels in Desc (minus any curried labels). // with those of the variable labels in Desc (minus any curried labels).
// //
// This method is used for the same purpose as // This method is used for the same purpose as
// GetMetricWithLabelValues(...string). See there for pros and cons of the two // GetMetricWithLabelValues(...string). See there for pros and cons of the two
// methods. // methods.
func (v *SummaryVec) GetMetricWith(labels Labels) (Observer, error) { func (v *SummaryVec) GetMetricWith(labels Labels) (Observer, error) {
metric, err := v.metricVec.getMetricWith(labels) metric, err := v.MetricVec.GetMetricWith(labels)
if metric != nil { if metric != nil {
return metric.(Observer), err return metric.(Observer), err
} }
...@@ -630,7 +637,7 @@ func (v *SummaryVec) With(labels Labels) Observer { ...@@ -630,7 +637,7 @@ func (v *SummaryVec) With(labels Labels) Observer {
// registered with a given registry (usually the uncurried version). The Reset // registered with a given registry (usually the uncurried version). The Reset
// method deletes all metrics, even if called on a curried vector. // method deletes all metrics, even if called on a curried vector.
func (v *SummaryVec) CurryWith(labels Labels) (ObserverVec, error) { func (v *SummaryVec) CurryWith(labels Labels) (ObserverVec, error) {
vec, err := v.curryWith(labels) vec, err := v.MetricVec.CurryWith(labels)
if vec != nil { if vec != nil {
return &SummaryVec{vec}, err return &SummaryVec{vec}, err
} }
...@@ -716,7 +723,7 @@ func NewConstSummary( ...@@ -716,7 +723,7 @@ func NewConstSummary(
count: count, count: count,
sum: sum, sum: sum,
quantiles: quantiles, quantiles: quantiles,
labelPairs: makeLabelPairs(desc, labelValues), labelPairs: MakeLabelPairs(desc, labelValues),
}, nil }, nil
} }
......
...@@ -19,7 +19,7 @@ import ( ...@@ -19,7 +19,7 @@ import (
"time" "time"
"unicode/utf8" "unicode/utf8"
//lint:ignore SA1019 Need to keep deprecated package for compatibility. //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes" "github.com/golang/protobuf/ptypes"
...@@ -63,7 +63,7 @@ func newValueFunc(desc *Desc, valueType ValueType, function func() float64) *val ...@@ -63,7 +63,7 @@ func newValueFunc(desc *Desc, valueType ValueType, function func() float64) *val
desc: desc, desc: desc,
valType: valueType, valType: valueType,
function: function, function: function,
labelPairs: makeLabelPairs(desc, nil), labelPairs: MakeLabelPairs(desc, nil),
} }
result.init(result) result.init(result)
return result return result
...@@ -95,7 +95,7 @@ func NewConstMetric(desc *Desc, valueType ValueType, value float64, labelValues ...@@ -95,7 +95,7 @@ func NewConstMetric(desc *Desc, valueType ValueType, value float64, labelValues
desc: desc, desc: desc,
valType: valueType, valType: valueType,
val: value, val: value,
labelPairs: makeLabelPairs(desc, labelValues), labelPairs: MakeLabelPairs(desc, labelValues),
}, nil }, nil
} }
...@@ -145,7 +145,14 @@ func populateMetric( ...@@ -145,7 +145,14 @@ func populateMetric(
return nil return nil
} }
func makeLabelPairs(desc *Desc, labelValues []string) []*dto.LabelPair { // MakeLabelPairs is a helper function to create protobuf LabelPairs from the
// variable and constant labels in the provided Desc. The values for the
// variable labels are defined by the labelValues slice, which must be in the
// same order as the corresponding variable labels in the Desc.
//
// This function is only needed for custom Metric implementations. See MetricVec
// example.
func MakeLabelPairs(desc *Desc, labelValues []string) []*dto.LabelPair {
totalLen := len(desc.variableLabels) + len(desc.constLabelPairs) totalLen := len(desc.variableLabels) + len(desc.constLabelPairs)
if totalLen == 0 { if totalLen == 0 {
// Super fast path. // Super fast path.
......
...@@ -20,12 +20,20 @@ import ( ...@@ -20,12 +20,20 @@ import (
"github.com/prometheus/common/model" "github.com/prometheus/common/model"
) )
// metricVec is a Collector to bundle metrics of the same name that differ in // MetricVec is a Collector to bundle metrics of the same name that differ in
// their label values. metricVec is not used directly (and therefore // their label values. MetricVec is not used directly but as a building block
// unexported). It is used as a building block for implementations of vectors of // for implementations of vectors of a given metric type, like GaugeVec,
// a given metric type, like GaugeVec, CounterVec, SummaryVec, and HistogramVec. // CounterVec, SummaryVec, and HistogramVec. It is exported so that it can be
// It also handles label currying. // used for custom Metric implementations.
type metricVec struct { //
// To create a FooVec for custom Metric Foo, embed a pointer to MetricVec in
// FooVec and initialize it with NewMetricVec. Implement wrappers for
// GetMetricWithLabelValues and GetMetricWith that return (Foo, error) rather
// than (Metric, error). Similarly, create a wrapper for CurryWith that returns
// (*FooVec, error) rather than (*MetricVec, error). It is recommended to also
// add the convenience methods WithLabelValues, With, and MustCurryWith, which
// panic instead of returning errors. See also the MetricVec example.
type MetricVec struct {
*metricMap *metricMap
curry []curriedLabelValue curry []curriedLabelValue
...@@ -35,9 +43,9 @@ type metricVec struct { ...@@ -35,9 +43,9 @@ type metricVec struct {
hashAddByte func(h uint64, b byte) uint64 hashAddByte func(h uint64, b byte) uint64
} }
// newMetricVec returns an initialized metricVec. // NewMetricVec returns an initialized metricVec.
func newMetricVec(desc *Desc, newMetric func(lvs ...string) Metric) *metricVec { func NewMetricVec(desc *Desc, newMetric func(lvs ...string) Metric) *MetricVec {
return &metricVec{ return &MetricVec{
metricMap: &metricMap{ metricMap: &metricMap{
metrics: map[uint64][]metricWithLabelValues{}, metrics: map[uint64][]metricWithLabelValues{},
desc: desc, desc: desc,
...@@ -63,7 +71,7 @@ func newMetricVec(desc *Desc, newMetric func(lvs ...string) Metric) *metricVec { ...@@ -63,7 +71,7 @@ func newMetricVec(desc *Desc, newMetric func(lvs ...string) Metric) *metricVec {
// latter has a much more readable (albeit more verbose) syntax, but it comes // latter has a much more readable (albeit more verbose) syntax, but it comes
// with a performance overhead (for creating and processing the Labels map). // with a performance overhead (for creating and processing the Labels map).
// See also the CounterVec example. // See also the CounterVec example.
func (m *metricVec) DeleteLabelValues(lvs ...string) bool { func (m *MetricVec) DeleteLabelValues(lvs ...string) bool {
h, err := m.hashLabelValues(lvs) h, err := m.hashLabelValues(lvs)
if err != nil { if err != nil {
return false return false
...@@ -82,7 +90,7 @@ func (m *metricVec) DeleteLabelValues(lvs ...string) bool { ...@@ -82,7 +90,7 @@ func (m *metricVec) DeleteLabelValues(lvs ...string) bool {
// //
// This method is used for the same purpose as DeleteLabelValues(...string). See // This method is used for the same purpose as DeleteLabelValues(...string). See
// there for pros and cons of the two methods. // there for pros and cons of the two methods.
func (m *metricVec) Delete(labels Labels) bool { func (m *MetricVec) Delete(labels Labels) bool {
h, err := m.hashLabels(labels) h, err := m.hashLabels(labels)
if err != nil { if err != nil {
return false return false
...@@ -95,15 +103,32 @@ func (m *metricVec) Delete(labels Labels) bool { ...@@ -95,15 +103,32 @@ func (m *metricVec) Delete(labels Labels) bool {
// show up in GoDoc. // show up in GoDoc.
// Describe implements Collector. // Describe implements Collector.
func (m *metricVec) Describe(ch chan<- *Desc) { m.metricMap.Describe(ch) } func (m *MetricVec) Describe(ch chan<- *Desc) { m.metricMap.Describe(ch) }
// Collect implements Collector. // Collect implements Collector.
func (m *metricVec) Collect(ch chan<- Metric) { m.metricMap.Collect(ch) } func (m *MetricVec) Collect(ch chan<- Metric) { m.metricMap.Collect(ch) }
// Reset deletes all metrics in this vector. // Reset deletes all metrics in this vector.
func (m *metricVec) Reset() { m.metricMap.Reset() } func (m *MetricVec) Reset() { m.metricMap.Reset() }
func (m *metricVec) curryWith(labels Labels) (*metricVec, error) { // CurryWith returns a vector curried with the provided labels, i.e. the
// returned vector has those labels pre-set for all labeled operations performed
// on it. The cardinality of the curried vector is reduced accordingly. The
// order of the remaining labels stays the same (just with the curried labels
// taken out of the sequence – which is relevant for the
// (GetMetric)WithLabelValues methods). It is possible to curry a curried
// vector, but only with labels not yet used for currying before.
//
// The metrics contained in the MetricVec are shared between the curried and
// uncurried vectors. They are just accessed differently. Curried and uncurried
// vectors behave identically in terms of collection. Only one must be
// registered with a given registry (usually the uncurried version). The Reset
// method deletes all metrics, even if called on a curried vector.
//
// Note that CurryWith is usually not called directly but through a wrapper
// around MetricVec, implementing a vector for a specific Metric
// implementation, for example GaugeVec.
func (m *MetricVec) CurryWith(labels Labels) (*MetricVec, error) {
var ( var (
newCurry []curriedLabelValue newCurry []curriedLabelValue
oldCurry = m.curry oldCurry = m.curry
...@@ -128,7 +153,7 @@ func (m *metricVec) curryWith(labels Labels) (*metricVec, error) { ...@@ -128,7 +153,7 @@ func (m *metricVec) curryWith(labels Labels) (*metricVec, error) {
return nil, fmt.Errorf("%d unknown label(s) found during currying", l) return nil, fmt.Errorf("%d unknown label(s) found during currying", l)
} }
return &metricVec{ return &MetricVec{
metricMap: m.metricMap, metricMap: m.metricMap,
curry: newCurry, curry: newCurry,
hashAdd: m.hashAdd, hashAdd: m.hashAdd,
...@@ -136,7 +161,34 @@ func (m *metricVec) curryWith(labels Labels) (*metricVec, error) { ...@@ -136,7 +161,34 @@ func (m *metricVec) curryWith(labels Labels) (*metricVec, error) {
}, nil }, nil
} }
func (m *metricVec) getMetricWithLabelValues(lvs ...string) (Metric, error) { // GetMetricWithLabelValues returns the Metric for the given slice of label
// values (same order as the variable labels in Desc). If that combination of
// label values is accessed for the first time, a new Metric is created (by
// calling the newMetric function provided during construction of the
// MetricVec).
//
// It is possible to call this method without using the returned Metric to only
// create the new Metric but leave it in its initial state.
//
// Keeping the Metric for later use is possible (and should be considered if
// performance is critical), but keep in mind that Reset, DeleteLabelValues and
// Delete can be used to delete the Metric from the MetricVec. In that case, the
// Metric will still exist, but it will not be exported anymore, even if a
// Metric with the same label values is created later.
//
// An error is returned if the number of label values is not the same as the
// number of variable labels in Desc (minus any curried labels).
//
// Note that for more than one label value, this method is prone to mistakes
// caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as
// an alternative to avoid that type of mistake. For higher label numbers, the
// latter has a much more readable (albeit more verbose) syntax, but it comes
// with a performance overhead (for creating and processing the Labels map).
//
// Note that GetMetricWithLabelValues is usually not called directly but through
// a wrapper around MetricVec, implementing a vector for a specific Metric
// implementation, for example GaugeVec.
func (m *MetricVec) GetMetricWithLabelValues(lvs ...string) (Metric, error) {
h, err := m.hashLabelValues(lvs) h, err := m.hashLabelValues(lvs)
if err != nil { if err != nil {
return nil, err return nil, err
...@@ -145,7 +197,23 @@ func (m *metricVec) getMetricWithLabelValues(lvs ...string) (Metric, error) { ...@@ -145,7 +197,23 @@ func (m *metricVec) getMetricWithLabelValues(lvs ...string) (Metric, error) {
return m.metricMap.getOrCreateMetricWithLabelValues(h, lvs, m.curry), nil return m.metricMap.getOrCreateMetricWithLabelValues(h, lvs, m.curry), nil
} }
func (m *metricVec) getMetricWith(labels Labels) (Metric, error) { // GetMetricWith returns the Metric for the given Labels map (the label names
// must match those of the variable labels in Desc). If that label map is
// accessed for the first time, a new Metric is created. Implications of
// creating a Metric without using it and keeping the Metric for later use
// are the same as for GetMetricWithLabelValues.
//
// An error is returned if the number and names of the Labels are inconsistent
// with those of the variable labels in Desc (minus any curried labels).
//
// This method is used for the same purpose as
// GetMetricWithLabelValues(...string). See there for pros and cons of the two
// methods.
//
// Note that GetMetricWith is usually not called directly but through a wrapper
// around MetricVec, implementing a vector for a specific Metric implementation,
// for example GaugeVec.
func (m *MetricVec) GetMetricWith(labels Labels) (Metric, error) {
h, err := m.hashLabels(labels) h, err := m.hashLabels(labels)
if err != nil { if err != nil {
return nil, err return nil, err
...@@ -154,7 +222,7 @@ func (m *metricVec) getMetricWith(labels Labels) (Metric, error) { ...@@ -154,7 +222,7 @@ func (m *metricVec) getMetricWith(labels Labels) (Metric, error) {
return m.metricMap.getOrCreateMetricWithLabels(h, labels, m.curry), nil return m.metricMap.getOrCreateMetricWithLabels(h, labels, m.curry), nil
} }
func (m *metricVec) hashLabelValues(vals []string) (uint64, error) { func (m *MetricVec) hashLabelValues(vals []string) (uint64, error) {
if err := validateLabelValues(vals, len(m.desc.variableLabels)-len(m.curry)); err != nil { if err := validateLabelValues(vals, len(m.desc.variableLabels)-len(m.curry)); err != nil {
return 0, err return 0, err
} }
...@@ -177,7 +245,7 @@ func (m *metricVec) hashLabelValues(vals []string) (uint64, error) { ...@@ -177,7 +245,7 @@ func (m *metricVec) hashLabelValues(vals []string) (uint64, error) {
return h, nil return h, nil
} }
func (m *metricVec) hashLabels(labels Labels) (uint64, error) { func (m *MetricVec) hashLabels(labels Labels) (uint64, error) {
if err := validateValuesInLabels(labels, len(m.desc.variableLabels)-len(m.curry)); err != nil { if err := validateValuesInLabels(labels, len(m.desc.variableLabels)-len(m.curry)); err != nil {
return 0, err return 0, err
} }
...@@ -276,7 +344,9 @@ func (m *metricMap) deleteByHashWithLabelValues( ...@@ -276,7 +344,9 @@ func (m *metricMap) deleteByHashWithLabelValues(
} }
if len(metrics) > 1 { if len(metrics) > 1 {
old := metrics
m.metrics[h] = append(metrics[:i], metrics[i+1:]...) m.metrics[h] = append(metrics[:i], metrics[i+1:]...)
old[len(old)-1] = metricWithLabelValues{}
} else { } else {
delete(m.metrics, h) delete(m.metrics, h)
} }
...@@ -302,7 +372,9 @@ func (m *metricMap) deleteByHashWithLabels( ...@@ -302,7 +372,9 @@ func (m *metricMap) deleteByHashWithLabels(
} }
if len(metrics) > 1 { if len(metrics) > 1 {
old := metrics
m.metrics[h] = append(metrics[:i], metrics[i+1:]...) m.metrics[h] = append(metrics[:i], metrics[i+1:]...)
old[len(old)-1] = metricWithLabelValues{}
} else { } else {
delete(m.metrics, h) delete(m.metrics, h)
} }
......
...@@ -17,7 +17,7 @@ import ( ...@@ -17,7 +17,7 @@ import (
"fmt" "fmt"
"sort" "sort"
//lint:ignore SA1019 Need to keep deprecated package for compatibility. //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
dto "github.com/prometheus/client_model/go" dto "github.com/prometheus/client_model/go"
...@@ -32,7 +32,9 @@ import ( ...@@ -32,7 +32,9 @@ import (
// in a no-op Registerer. // in a no-op Registerer.
// //
// WrapRegistererWith provides a way to add fixed labels to a subset of // WrapRegistererWith provides a way to add fixed labels to a subset of
// Collectors. It should not be used to add fixed labels to all metrics exposed. // Collectors. It should not be used to add fixed labels to all metrics
// exposed. See also
// https://prometheus.io/docs/instrumenting/writing_exporters/#target-labels-not-static-scraped-labels
// //
// Conflicts between Collectors registered through the original Registerer with // Conflicts between Collectors registered through the original Registerer with
// Collectors registered through the wrapping Registerer will still be // Collectors registered through the wrapping Registerer will still be
......
...@@ -164,7 +164,7 @@ func (sd *SampleDecoder) Decode(s *model.Vector) error { ...@@ -164,7 +164,7 @@ func (sd *SampleDecoder) Decode(s *model.Vector) error {
} }
// ExtractSamples builds a slice of samples from the provided metric // ExtractSamples builds a slice of samples from the provided metric
// families. If an error occurrs during sample extraction, it continues to // families. If an error occurs during sample extraction, it continues to
// extract from the remaining metric families. The returned error is the last // extract from the remaining metric families. The returned error is the last
// error that has occurred. // error that has occurred.
func ExtractSamples(o *DecodeOptions, fams ...*dto.MetricFamily) (model.Vector, error) { func ExtractSamples(o *DecodeOptions, fams ...*dto.MetricFamily) (model.Vector, error) {
......
...@@ -299,6 +299,17 @@ func (p *TextParser) startLabelName() stateFn { ...@@ -299,6 +299,17 @@ func (p *TextParser) startLabelName() stateFn {
p.parseError(fmt.Sprintf("expected '=' after label name, found %q", p.currentByte)) p.parseError(fmt.Sprintf("expected '=' after label name, found %q", p.currentByte))
return nil return nil
} }
// Check for duplicate label names.
labels := make(map[string]struct{})
for _, l := range p.currentMetric.Label {
lName := l.GetName()
if _, exists := labels[lName]; !exists {
labels[lName] = struct{}{}
} else {
p.parseError(fmt.Sprintf("duplicate label names for metric %q", p.currentMF.GetName()))
return nil
}
}
return p.startLabelValue return p.startLabelValue
} }
......
...@@ -20,7 +20,7 @@ const ( ...@@ -20,7 +20,7 @@ const (
prime64 = 1099511628211 prime64 = 1099511628211
) )
// hashNew initializies a new fnv64a hash value. // hashNew initializes a new fnv64a hash value.
func hashNew() uint64 { func hashNew() uint64 {
return offset64 return offset64
} }
......
...@@ -45,6 +45,14 @@ const ( ...@@ -45,6 +45,14 @@ const (
// scrape a target. // scrape a target.
MetricsPathLabel = "__metrics_path__" MetricsPathLabel = "__metrics_path__"
// ScrapeIntervalLabel is the name of the label that holds the scrape interval
// used to scrape a target.
ScrapeIntervalLabel = "__scrape_interval__"
// ScrapeTimeoutLabel is the name of the label that holds the scrape
// timeout used to scrape a target.
ScrapeTimeoutLabel = "__scrape_timeout__"
// ReservedLabelPrefix is a prefix which is not legal in user-supplied // ReservedLabelPrefix is a prefix which is not legal in user-supplied
// label names. // label names.
ReservedLabelPrefix = "__" ReservedLabelPrefix = "__"
......
...@@ -14,6 +14,8 @@ ...@@ -14,6 +14,8 @@
package model package model
import ( import (
"encoding/json"
"errors"
"fmt" "fmt"
"math" "math"
"regexp" "regexp"
...@@ -181,77 +183,118 @@ func (d *Duration) Type() string { ...@@ -181,77 +183,118 @@ func (d *Duration) Type() string {
return "duration" return "duration"
} }
var durationRE = regexp.MustCompile("^([0-9]+)(y|w|d|h|m|s|ms)$") var durationRE = regexp.MustCompile("^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$")
// ParseDuration parses a string into a time.Duration, assuming that a year // ParseDuration parses a string into a time.Duration, assuming that a year
// always has 365d, a week always has 7d, and a day always has 24h. // always has 365d, a week always has 7d, and a day always has 24h.
func ParseDuration(durationStr string) (Duration, error) { func ParseDuration(durationStr string) (Duration, error) {
// Allow 0 without a unit. switch durationStr {
if durationStr == "0" { case "0":
// Allow 0 without a unit.
return 0, nil return 0, nil
case "":
return 0, fmt.Errorf("empty duration string")
} }
matches := durationRE.FindStringSubmatch(durationStr) matches := durationRE.FindStringSubmatch(durationStr)
if len(matches) != 3 { if matches == nil {
return 0, fmt.Errorf("not a valid duration string: %q", durationStr) return 0, fmt.Errorf("not a valid duration string: %q", durationStr)
} }
var ( var dur time.Duration
n, _ = strconv.Atoi(matches[1])
dur = time.Duration(n) * time.Millisecond // Parse the match at pos `pos` in the regex and use `mult` to turn that
) // into ms, then add that value to the total parsed duration.
switch unit := matches[2]; unit { var overflowErr error
case "y": m := func(pos int, mult time.Duration) {
dur *= 1000 * 60 * 60 * 24 * 365 if matches[pos] == "" {
case "w": return
dur *= 1000 * 60 * 60 * 24 * 7 }
case "d": n, _ := strconv.Atoi(matches[pos])
dur *= 1000 * 60 * 60 * 24
case "h": // Check if the provided duration overflows time.Duration (> ~ 290years).
dur *= 1000 * 60 * 60 if n > int((1<<63-1)/mult/time.Millisecond) {
case "m": overflowErr = errors.New("duration out of range")
dur *= 1000 * 60 }
case "s": d := time.Duration(n) * time.Millisecond
dur *= 1000 dur += d * mult
case "ms":
// Value already correct if dur < 0 {
default: overflowErr = errors.New("duration out of range")
return 0, fmt.Errorf("invalid time unit in duration string: %q", unit) }
} }
return Duration(dur), nil
m(2, 1000*60*60*24*365) // y
m(4, 1000*60*60*24*7) // w
m(6, 1000*60*60*24) // d
m(8, 1000*60*60) // h
m(10, 1000*60) // m
m(12, 1000) // s
m(14, 1) // ms
return Duration(dur), overflowErr
} }
func (d Duration) String() string { func (d Duration) String() string {
var ( var (
ms = int64(time.Duration(d) / time.Millisecond) ms = int64(time.Duration(d) / time.Millisecond)
unit = "ms" r = ""
) )
if ms == 0 { if ms == 0 {
return "0s" return "0s"
} }
factors := map[string]int64{
"y": 1000 * 60 * 60 * 24 * 365, f := func(unit string, mult int64, exact bool) {
"w": 1000 * 60 * 60 * 24 * 7, if exact && ms%mult != 0 {
"d": 1000 * 60 * 60 * 24, return
"h": 1000 * 60 * 60, }
"m": 1000 * 60, if v := ms / mult; v > 0 {
"s": 1000, r += fmt.Sprintf("%d%s", v, unit)
"ms": 1, ms -= v * mult
}
} }
switch int64(0) { // Only format years and weeks if the remainder is zero, as it is often
case ms % factors["y"]: // easier to read 90d than 12w6d.
unit = "y" f("y", 1000*60*60*24*365, true)
case ms % factors["w"]: f("w", 1000*60*60*24*7, true)
unit = "w"
case ms % factors["d"]: f("d", 1000*60*60*24, false)
unit = "d" f("h", 1000*60*60, false)
case ms % factors["h"]: f("m", 1000*60, false)
unit = "h" f("s", 1000, false)
case ms % factors["m"]: f("ms", 1, false)
unit = "m"
case ms % factors["s"]: return r
unit = "s" }
// MarshalJSON implements the json.Marshaler interface.
func (d Duration) MarshalJSON() ([]byte, error) {
return json.Marshal(d.String())
}
// UnmarshalJSON implements the json.Unmarshaler interface.
func (d *Duration) UnmarshalJSON(bytes []byte) error {
var s string
if err := json.Unmarshal(bytes, &s); err != nil {
return err
}
dur, err := ParseDuration(s)
if err != nil {
return err
} }
return fmt.Sprintf("%v%v", ms/factors[unit], unit) *d = dur
return nil
}
// MarshalText implements the encoding.TextMarshaler interface.
func (d *Duration) MarshalText() ([]byte, error) {
return []byte(d.String()), nil
}
// UnmarshalText implements the encoding.TextUnmarshaler interface.
func (d *Duration) UnmarshalText(text []byte) error {
var err error
*d, err = ParseDuration(string(text))
return err
} }
// MarshalYAML implements the yaml.Marshaler interface. // MarshalYAML implements the yaml.Marshaler interface.
......
...@@ -78,7 +78,7 @@ ifneq ($(shell which gotestsum),) ...@@ -78,7 +78,7 @@ ifneq ($(shell which gotestsum),)
endif endif
endif endif
PROMU_VERSION ?= 0.5.0 PROMU_VERSION ?= 0.7.0
PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
GOLANGCI_LINT := GOLANGCI_LINT :=
...@@ -245,10 +245,12 @@ common-docker-publish: $(PUBLISH_DOCKER_ARCHS) ...@@ -245,10 +245,12 @@ common-docker-publish: $(PUBLISH_DOCKER_ARCHS)
$(PUBLISH_DOCKER_ARCHS): common-docker-publish-%: $(PUBLISH_DOCKER_ARCHS): common-docker-publish-%:
docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)"
DOCKER_MAJOR_VERSION_TAG = $(firstword $(subst ., ,$(shell cat VERSION)))
.PHONY: common-docker-tag-latest $(TAG_DOCKER_ARCHS) .PHONY: common-docker-tag-latest $(TAG_DOCKER_ARCHS)
common-docker-tag-latest: $(TAG_DOCKER_ARCHS) common-docker-tag-latest: $(TAG_DOCKER_ARCHS)
$(TAG_DOCKER_ARCHS): common-docker-tag-latest-%: $(TAG_DOCKER_ARCHS): common-docker-tag-latest-%:
docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest" docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest"
docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)"
.PHONY: common-docker-manifest .PHONY: common-docker-manifest
common-docker-manifest: common-docker-manifest:
......
# Reporting a security issue
The Prometheus security policy, including how to report vulnerabilities, can be
found here:
https://prometheus.io/docs/operating/security/
...@@ -36,7 +36,7 @@ type ARPEntry struct { ...@@ -36,7 +36,7 @@ type ARPEntry struct {
func (fs FS) GatherARPEntries() ([]ARPEntry, error) { func (fs FS) GatherARPEntries() ([]ARPEntry, error) {
data, err := ioutil.ReadFile(fs.proc.Path("net/arp")) data, err := ioutil.ReadFile(fs.proc.Path("net/arp"))
if err != nil { if err != nil {
return nil, fmt.Errorf("error reading arp %s: %s", fs.proc.Path("net/arp"), err) return nil, fmt.Errorf("error reading arp %q: %w", fs.proc.Path("net/arp"), err)
} }
return parseARPEntries(data) return parseARPEntries(data)
...@@ -59,7 +59,7 @@ func parseARPEntries(data []byte) ([]ARPEntry, error) { ...@@ -59,7 +59,7 @@ func parseARPEntries(data []byte) ([]ARPEntry, error) {
} else if width == expectedDataWidth { } else if width == expectedDataWidth {
entry, err := parseARPEntry(columns) entry, err := parseARPEntry(columns)
if err != nil { if err != nil {
return []ARPEntry{}, fmt.Errorf("failed to parse ARP entry: %s", err) return []ARPEntry{}, fmt.Errorf("failed to parse ARP entry: %w", err)
} }
entries = append(entries, entry) entries = append(entries, entry)
} else { } else {
......
...@@ -74,7 +74,7 @@ func parseBuddyInfo(r io.Reader) ([]BuddyInfo, error) { ...@@ -74,7 +74,7 @@ func parseBuddyInfo(r io.Reader) ([]BuddyInfo, error) {
for i := 0; i < arraySize; i++ { for i := 0; i < arraySize; i++ {
sizes[i], err = strconv.ParseFloat(parts[i+4], 64) sizes[i], err = strconv.ParseFloat(parts[i+4], 64)
if err != nil { if err != nil {
return nil, fmt.Errorf("invalid value in buddyinfo: %s", err) return nil, fmt.Errorf("invalid value in buddyinfo: %w", err)
} }
} }
......
...@@ -19,6 +19,7 @@ import ( ...@@ -19,6 +19,7 @@ import (
"bufio" "bufio"
"bytes" "bytes"
"errors" "errors"
"fmt"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
...@@ -77,7 +78,7 @@ func parseCPUInfoX86(info []byte) ([]CPUInfo, error) { ...@@ -77,7 +78,7 @@ func parseCPUInfoX86(info []byte) ([]CPUInfo, error) {
// find the first "processor" line // find the first "processor" line
firstLine := firstNonEmptyLine(scanner) firstLine := firstNonEmptyLine(scanner)
if !strings.HasPrefix(firstLine, "processor") || !strings.Contains(firstLine, ":") { if !strings.HasPrefix(firstLine, "processor") || !strings.Contains(firstLine, ":") {
return nil, errors.New("invalid cpuinfo file: " + firstLine) return nil, fmt.Errorf("invalid cpuinfo file: %q", firstLine)
} }
field := strings.SplitN(firstLine, ": ", 2) field := strings.SplitN(firstLine, ": ", 2)
v, err := strconv.ParseUint(field[1], 0, 32) v, err := strconv.ParseUint(field[1], 0, 32)
...@@ -192,7 +193,7 @@ func parseCPUInfoARM(info []byte) ([]CPUInfo, error) { ...@@ -192,7 +193,7 @@ func parseCPUInfoARM(info []byte) ([]CPUInfo, error) {
firstLine := firstNonEmptyLine(scanner) firstLine := firstNonEmptyLine(scanner)
match, _ := regexp.MatchString("^[Pp]rocessor", firstLine) match, _ := regexp.MatchString("^[Pp]rocessor", firstLine)
if !match || !strings.Contains(firstLine, ":") { if !match || !strings.Contains(firstLine, ":") {
return nil, errors.New("invalid cpuinfo file: " + firstLine) return nil, fmt.Errorf("invalid cpuinfo file: %q", firstLine)
} }
field := strings.SplitN(firstLine, ": ", 2) field := strings.SplitN(firstLine, ": ", 2)
cpuinfo := []CPUInfo{} cpuinfo := []CPUInfo{}
...@@ -256,7 +257,7 @@ func parseCPUInfoS390X(info []byte) ([]CPUInfo, error) { ...@@ -256,7 +257,7 @@ func parseCPUInfoS390X(info []byte) ([]CPUInfo, error) {
firstLine := firstNonEmptyLine(scanner) firstLine := firstNonEmptyLine(scanner)
if !strings.HasPrefix(firstLine, "vendor_id") || !strings.Contains(firstLine, ":") { if !strings.HasPrefix(firstLine, "vendor_id") || !strings.Contains(firstLine, ":") {
return nil, errors.New("invalid cpuinfo file: " + firstLine) return nil, fmt.Errorf("invalid cpuinfo file: %q", firstLine)
} }
field := strings.SplitN(firstLine, ": ", 2) field := strings.SplitN(firstLine, ": ", 2)
cpuinfo := []CPUInfo{} cpuinfo := []CPUInfo{}
...@@ -281,7 +282,7 @@ func parseCPUInfoS390X(info []byte) ([]CPUInfo, error) { ...@@ -281,7 +282,7 @@ func parseCPUInfoS390X(info []byte) ([]CPUInfo, error) {
if strings.HasPrefix(line, "processor") { if strings.HasPrefix(line, "processor") {
match := cpuinfoS390XProcessorRegexp.FindStringSubmatch(line) match := cpuinfoS390XProcessorRegexp.FindStringSubmatch(line)
if len(match) < 2 { if len(match) < 2 {
return nil, errors.New("Invalid line found in cpuinfo: " + line) return nil, fmt.Errorf("invalid cpuinfo file: %q", firstLine)
} }
cpu := commonCPUInfo cpu := commonCPUInfo
v, err := strconv.ParseUint(match[1], 0, 32) v, err := strconv.ParseUint(match[1], 0, 32)
...@@ -313,6 +314,22 @@ func parseCPUInfoS390X(info []byte) ([]CPUInfo, error) { ...@@ -313,6 +314,22 @@ func parseCPUInfoS390X(info []byte) ([]CPUInfo, error) {
return nil, err return nil, err
} }
cpuinfo[i].CPUMHz = v cpuinfo[i].CPUMHz = v
case "physical id":
cpuinfo[i].PhysicalID = field[1]
case "core id":
cpuinfo[i].CoreID = field[1]
case "cpu cores":
v, err := strconv.ParseUint(field[1], 0, 32)
if err != nil {
return nil, err
}
cpuinfo[i].CPUCores = uint(v)
case "siblings":
v, err := strconv.ParseUint(field[1], 0, 32)
if err != nil {
return nil, err
}
cpuinfo[i].Siblings = uint(v)
} }
} }
...@@ -325,7 +342,7 @@ func parseCPUInfoMips(info []byte) ([]CPUInfo, error) { ...@@ -325,7 +342,7 @@ func parseCPUInfoMips(info []byte) ([]CPUInfo, error) {
// find the first "processor" line // find the first "processor" line
firstLine := firstNonEmptyLine(scanner) firstLine := firstNonEmptyLine(scanner)
if !strings.HasPrefix(firstLine, "system type") || !strings.Contains(firstLine, ":") { if !strings.HasPrefix(firstLine, "system type") || !strings.Contains(firstLine, ":") {
return nil, errors.New("invalid cpuinfo file: " + firstLine) return nil, fmt.Errorf("invalid cpuinfo file: %q", firstLine)
} }
field := strings.SplitN(firstLine, ": ", 2) field := strings.SplitN(firstLine, ": ", 2)
cpuinfo := []CPUInfo{} cpuinfo := []CPUInfo{}
...@@ -367,7 +384,7 @@ func parseCPUInfoPPC(info []byte) ([]CPUInfo, error) { ...@@ -367,7 +384,7 @@ func parseCPUInfoPPC(info []byte) ([]CPUInfo, error) {
firstLine := firstNonEmptyLine(scanner) firstLine := firstNonEmptyLine(scanner)
if !strings.HasPrefix(firstLine, "processor") || !strings.Contains(firstLine, ":") { if !strings.HasPrefix(firstLine, "processor") || !strings.Contains(firstLine, ":") {
return nil, errors.New("invalid cpuinfo file: " + firstLine) return nil, fmt.Errorf("invalid cpuinfo file: %q", firstLine)
} }
field := strings.SplitN(firstLine, ": ", 2) field := strings.SplitN(firstLine, ": ", 2)
v, err := strconv.ParseUint(field[1], 0, 32) v, err := strconv.ParseUint(field[1], 0, 32)
...@@ -412,7 +429,7 @@ func parseCPUInfoRISCV(info []byte) ([]CPUInfo, error) { ...@@ -412,7 +429,7 @@ func parseCPUInfoRISCV(info []byte) ([]CPUInfo, error) {
firstLine := firstNonEmptyLine(scanner) firstLine := firstNonEmptyLine(scanner)
if !strings.HasPrefix(firstLine, "processor") || !strings.Contains(firstLine, ":") { if !strings.HasPrefix(firstLine, "processor") || !strings.Contains(firstLine, ":") {
return nil, errors.New("invalid cpuinfo file: " + firstLine) return nil, fmt.Errorf("invalid cpuinfo file: %q", firstLine)
} }
field := strings.SplitN(firstLine, ": ", 2) field := strings.SplitN(firstLine, ": ", 2)
v, err := strconv.ParseUint(field[1], 0, 32) v, err := strconv.ParseUint(field[1], 0, 32)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment