Commit d462e1e8 authored by Guoliang Wang's avatar Guoliang Wang

Update vendor package github.com/coreos/...

parent c0974d73
...@@ -36,39 +36,39 @@ ...@@ -36,39 +36,39 @@
}, },
{ {
"ImportPath": "github.com/coreos/etcd/auth/authpb", "ImportPath": "github.com/coreos/etcd/auth/authpb",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/clientv3", "ImportPath": "github.com/coreos/etcd/clientv3",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes", "ImportPath": "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/etcdserver/etcdserverpb", "ImportPath": "github.com/coreos/etcd/etcdserver/etcdserverpb",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/mvcc/mvccpb", "ImportPath": "github.com/coreos/etcd/mvcc/mvccpb",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/pkg/tlsutil", "ImportPath": "github.com/coreos/etcd/pkg/tlsutil",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/pkg/transport", "ImportPath": "github.com/coreos/etcd/pkg/transport",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/pkg/types", "ImportPath": "github.com/coreos/etcd/pkg/types",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/go-systemd/daemon", "ImportPath": "github.com/coreos/go-systemd/daemon",
"Rev": "48702e0da86bd25e76cfef347e2adeb434a0d0a6" "Rev": "39ca1b05acc7ad1220e09f133283b8859a8b71ab"
}, },
{ {
"ImportPath": "github.com/davecgh/go-spew/spew", "ImportPath": "github.com/davecgh/go-spew/spew",
......
...@@ -36,39 +36,39 @@ ...@@ -36,39 +36,39 @@
}, },
{ {
"ImportPath": "github.com/coreos/etcd/auth/authpb", "ImportPath": "github.com/coreos/etcd/auth/authpb",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/clientv3", "ImportPath": "github.com/coreos/etcd/clientv3",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes", "ImportPath": "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/etcdserver/etcdserverpb", "ImportPath": "github.com/coreos/etcd/etcdserver/etcdserverpb",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/mvcc/mvccpb", "ImportPath": "github.com/coreos/etcd/mvcc/mvccpb",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/pkg/tlsutil", "ImportPath": "github.com/coreos/etcd/pkg/tlsutil",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/pkg/transport", "ImportPath": "github.com/coreos/etcd/pkg/transport",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/etcd/pkg/types", "ImportPath": "github.com/coreos/etcd/pkg/types",
"Rev": "fca8add78a9d926166eb739b8e4a124434025ba3" "Rev": "27fc7e2296f506182f58ce846e48f36b34fe6842"
}, },
{ {
"ImportPath": "github.com/coreos/go-systemd/daemon", "ImportPath": "github.com/coreos/go-systemd/daemon",
"Rev": "48702e0da86bd25e76cfef347e2adeb434a0d0a6" "Rev": "39ca1b05acc7ad1220e09f133283b8859a8b71ab"
}, },
{ {
"ImportPath": "github.com/davecgh/go-spew/spew", "ImportPath": "github.com/davecgh/go-spew/spew",
......
...@@ -265,6 +265,7 @@ filegroup( ...@@ -265,6 +265,7 @@ filegroup(
"//vendor/github.com/gophercloud/gophercloud:all-srcs", "//vendor/github.com/gophercloud/gophercloud:all-srcs",
"//vendor/github.com/gorilla/websocket:all-srcs", "//vendor/github.com/gorilla/websocket:all-srcs",
"//vendor/github.com/gregjones/httpcache:all-srcs", "//vendor/github.com/gregjones/httpcache:all-srcs",
"//vendor/github.com/grpc-ecosystem/go-grpc-middleware:all-srcs",
"//vendor/github.com/grpc-ecosystem/go-grpc-prometheus:all-srcs", "//vendor/github.com/grpc-ecosystem/go-grpc-prometheus:all-srcs",
"//vendor/github.com/grpc-ecosystem/grpc-gateway/runtime:all-srcs", "//vendor/github.com/grpc-ecosystem/grpc-gateway/runtime:all-srcs",
"//vendor/github.com/grpc-ecosystem/grpc-gateway/utilities:all-srcs", "//vendor/github.com/grpc-ecosystem/grpc-gateway/utilities:all-srcs",
...@@ -368,6 +369,9 @@ filegroup( ...@@ -368,6 +369,9 @@ filegroup(
"//vendor/github.com/vmware/photon-controller-go-sdk/photon:all-srcs", "//vendor/github.com/vmware/photon-controller-go-sdk/photon:all-srcs",
"//vendor/github.com/xanzy/go-cloudstack/cloudstack:all-srcs", "//vendor/github.com/xanzy/go-cloudstack/cloudstack:all-srcs",
"//vendor/github.com/xiang90/probing:all-srcs", "//vendor/github.com/xiang90/probing:all-srcs",
"//vendor/go.uber.org/atomic:all-srcs",
"//vendor/go.uber.org/multierr:all-srcs",
"//vendor/go.uber.org/zap:all-srcs",
"//vendor/golang.org/x/crypto/bcrypt:all-srcs", "//vendor/golang.org/x/crypto/bcrypt:all-srcs",
"//vendor/golang.org/x/crypto/blowfish:all-srcs", "//vendor/golang.org/x/crypto/blowfish:all-srcs",
"//vendor/golang.org/x/crypto/cryptobyte:all-srcs", "//vendor/golang.org/x/crypto/cryptobyte:all-srcs",
......
...@@ -68,11 +68,10 @@ func (m *Mutex) Lock(ctx context.Context) error { ...@@ -68,11 +68,10 @@ func (m *Mutex) Lock(ctx context.Context) error {
// wait for deletion revisions prior to myKey // wait for deletion revisions prior to myKey
hdr, werr := waitDeletes(ctx, client, m.pfx, m.myRev-1) hdr, werr := waitDeletes(ctx, client, m.pfx, m.myRev-1)
// release lock key if cancelled // release lock key if wait failed
select { if werr != nil {
case <-ctx.Done():
m.Unlock(client.Ctx()) m.Unlock(client.Ctx())
default: } else {
m.hdr = hdr m.hdr = hdr
} }
return werr return werr
......
...@@ -23,6 +23,7 @@ go_library( ...@@ -23,6 +23,7 @@ go_library(
"//vendor/github.com/coreos/etcd/rafthttp:go_default_library", "//vendor/github.com/coreos/etcd/rafthttp:go_default_library",
"//vendor/github.com/coreos/etcd/version:go_default_library", "//vendor/github.com/coreos/etcd/version:go_default_library",
"//vendor/github.com/coreos/pkg/capnslog:go_default_library", "//vendor/github.com/coreos/pkg/capnslog:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library",
], ],
) )
......
...@@ -24,6 +24,7 @@ import ( ...@@ -24,6 +24,7 @@ import (
"github.com/coreos/etcd/etcdserver/etcdserverpb" "github.com/coreos/etcd/etcdserver/etcdserverpb"
"github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
) )
...@@ -43,11 +44,6 @@ func HandlePrometheus(mux *http.ServeMux) { ...@@ -43,11 +44,6 @@ func HandlePrometheus(mux *http.ServeMux) {
mux.Handle(pathMetrics, promhttp.Handler()) mux.Handle(pathMetrics, promhttp.Handler())
} }
// HandleHealth registers health handler on '/health'.
func HandleHealth(mux *http.ServeMux, srv etcdserver.ServerV2) {
mux.Handle(PathHealth, NewHealthHandler(func() Health { return checkHealth(srv) }))
}
// NewHealthHandler handles '/health' requests. // NewHealthHandler handles '/health' requests.
func NewHealthHandler(hfunc func() Health) http.HandlerFunc { func NewHealthHandler(hfunc func() Health) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
...@@ -67,6 +63,26 @@ func NewHealthHandler(hfunc func() Health) http.HandlerFunc { ...@@ -67,6 +63,26 @@ func NewHealthHandler(hfunc func() Health) http.HandlerFunc {
} }
} }
var (
healthSuccess = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "etcd",
Subsystem: "server",
Name: "health_success",
Help: "The total number of successful health checks",
})
healthFailed = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "etcd",
Subsystem: "server",
Name: "health_failures",
Help: "The total number of failed health checks",
})
)
func init() {
prometheus.MustRegister(healthSuccess)
prometheus.MustRegister(healthFailed)
}
// Health defines etcd server health status. // Health defines etcd server health status.
// TODO: remove manual parsing in etcdctl cluster-health // TODO: remove manual parsing in etcdctl cluster-health
type Health struct { type Health struct {
...@@ -97,5 +113,11 @@ func checkHealth(srv etcdserver.ServerV2) Health { ...@@ -97,5 +113,11 @@ func checkHealth(srv etcdserver.ServerV2) Health {
h.Health = "false" h.Health = "false"
} }
} }
if h.Health == "true" {
healthSuccess.Inc()
} else {
healthFailed.Inc()
}
return h return h
} }
...@@ -37,15 +37,17 @@ go_library( ...@@ -37,15 +37,17 @@ go_library(
"//vendor/github.com/coreos/etcd/version:go_default_library", "//vendor/github.com/coreos/etcd/version:go_default_library",
"//vendor/github.com/coreos/pkg/capnslog:go_default_library", "//vendor/github.com/coreos/pkg/capnslog:go_default_library",
"//vendor/github.com/gogo/protobuf/proto:go_default_library", "//vendor/github.com/gogo/protobuf/proto:go_default_library",
"//vendor/github.com/grpc-ecosystem/go-grpc-middleware:go_default_library",
"//vendor/github.com/grpc-ecosystem/go-grpc-prometheus:go_default_library", "//vendor/github.com/grpc-ecosystem/go-grpc-prometheus:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/go.uber.org/zap:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library", "//vendor/google.golang.org/grpc:go_default_library",
"//vendor/google.golang.org/grpc/codes:go_default_library", "//vendor/google.golang.org/grpc/codes:go_default_library",
"//vendor/google.golang.org/grpc/credentials:go_default_library", "//vendor/google.golang.org/grpc/credentials:go_default_library",
"//vendor/google.golang.org/grpc/grpclog:go_default_library",
"//vendor/google.golang.org/grpc/health:go_default_library", "//vendor/google.golang.org/grpc/health:go_default_library",
"//vendor/google.golang.org/grpc/health/grpc_health_v1:go_default_library", "//vendor/google.golang.org/grpc/health/grpc_health_v1:go_default_library",
"//vendor/google.golang.org/grpc/metadata:go_default_library", "//vendor/google.golang.org/grpc/metadata:go_default_library",
"//vendor/google.golang.org/grpc/peer:go_default_library",
"//vendor/google.golang.org/grpc/status:go_default_library", "//vendor/google.golang.org/grpc/status:go_default_library",
], ],
) )
......
...@@ -16,18 +16,15 @@ package v3rpc ...@@ -16,18 +16,15 @@ package v3rpc
import ( import (
"crypto/tls" "crypto/tls"
"io/ioutil"
"math" "math"
"os"
"sync"
"github.com/coreos/etcd/etcdserver" "github.com/coreos/etcd/etcdserver"
pb "github.com/coreos/etcd/etcdserver/etcdserverpb" pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
"github.com/grpc-ecosystem/go-grpc-middleware"
"github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/grpc-ecosystem/go-grpc-prometheus"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials"
"google.golang.org/grpc/grpclog"
"google.golang.org/grpc/health" "google.golang.org/grpc/health"
healthpb "google.golang.org/grpc/health/grpc_health_v1" healthpb "google.golang.org/grpc/health/grpc_health_v1"
) )
...@@ -38,17 +35,21 @@ const ( ...@@ -38,17 +35,21 @@ const (
maxSendBytes = math.MaxInt32 maxSendBytes = math.MaxInt32
) )
// integration tests call this multiple times, which is racey in gRPC side
var grpclogOnce sync.Once
func Server(s *etcdserver.EtcdServer, tls *tls.Config, gopts ...grpc.ServerOption) *grpc.Server { func Server(s *etcdserver.EtcdServer, tls *tls.Config, gopts ...grpc.ServerOption) *grpc.Server {
var opts []grpc.ServerOption var opts []grpc.ServerOption
opts = append(opts, grpc.CustomCodec(&codec{})) opts = append(opts, grpc.CustomCodec(&codec{}))
if tls != nil { if tls != nil {
opts = append(opts, grpc.Creds(credentials.NewTLS(tls))) opts = append(opts, grpc.Creds(credentials.NewTLS(tls)))
} }
opts = append(opts, grpc.UnaryInterceptor(newUnaryInterceptor(s))) opts = append(opts, grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
opts = append(opts, grpc.StreamInterceptor(newStreamInterceptor(s))) newLogUnaryInterceptor(s),
newUnaryInterceptor(s),
grpc_prometheus.UnaryServerInterceptor,
)))
opts = append(opts, grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
newStreamInterceptor(s),
grpc_prometheus.StreamServerInterceptor,
)))
opts = append(opts, grpc.MaxRecvMsgSize(int(s.Cfg.MaxRequestBytes+grpcOverheadBytes))) opts = append(opts, grpc.MaxRecvMsgSize(int(s.Cfg.MaxRequestBytes+grpcOverheadBytes)))
opts = append(opts, grpc.MaxSendMsgSize(maxSendBytes)) opts = append(opts, grpc.MaxSendMsgSize(maxSendBytes))
opts = append(opts, grpc.MaxConcurrentStreams(maxStreams)) opts = append(opts, grpc.MaxConcurrentStreams(maxStreams))
...@@ -71,16 +72,5 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, gopts ...grpc.ServerOptio ...@@ -71,16 +72,5 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, gopts ...grpc.ServerOptio
// set zero values for metrics registered for this grpc server // set zero values for metrics registered for this grpc server
grpc_prometheus.Register(grpcServer) grpc_prometheus.Register(grpcServer)
grpclogOnce.Do(func() {
if s.Cfg.Debug {
grpc.EnableTracing = true
// enable info, warning, error
grpclog.SetLoggerV2(grpclog.NewLoggerV2(os.Stderr, os.Stderr, os.Stderr))
} else {
// only discard info
grpclog.SetLoggerV2(grpclog.NewLoggerV2(ioutil.Discard, os.Stderr, os.Stderr))
}
})
return grpcServer return grpcServer
} }
...@@ -25,9 +25,11 @@ import ( ...@@ -25,9 +25,11 @@ import (
"github.com/coreos/etcd/pkg/types" "github.com/coreos/etcd/pkg/types"
"github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft"
prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
"go.uber.org/zap"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/metadata" "google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
) )
const ( const (
...@@ -40,7 +42,7 @@ type streamsMap struct { ...@@ -40,7 +42,7 @@ type streamsMap struct {
} }
func newUnaryInterceptor(s *etcdserver.EtcdServer) grpc.UnaryServerInterceptor { func newUnaryInterceptor(s *etcdserver.EtcdServer) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
if !api.IsCapabilityEnabled(api.V3rpcCapability) { if !api.IsCapabilityEnabled(api.V3rpcCapability) {
return nil, rpctypes.ErrGRPCNotCapable return nil, rpctypes.ErrGRPCNotCapable
} }
...@@ -54,7 +56,124 @@ func newUnaryInterceptor(s *etcdserver.EtcdServer) grpc.UnaryServerInterceptor { ...@@ -54,7 +56,124 @@ func newUnaryInterceptor(s *etcdserver.EtcdServer) grpc.UnaryServerInterceptor {
} }
} }
return prometheus.UnaryServerInterceptor(ctx, req, info, handler) return handler(ctx, req)
}
}
func newLogUnaryInterceptor(s *etcdserver.EtcdServer) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
startTime := time.Now()
resp, err := handler(ctx, req)
defer logUnaryRequestStats(ctx, nil, info, startTime, req, resp)
return resp, err
}
}
func logUnaryRequestStats(ctx context.Context, lg *zap.Logger, info *grpc.UnaryServerInfo, startTime time.Time, req interface{}, resp interface{}) {
duration := time.Since(startTime)
remote := "No remote client info."
peerInfo, ok := peer.FromContext(ctx)
if ok {
remote = peerInfo.Addr.String()
}
var responseType string = info.FullMethod
var reqCount, respCount int64
var reqSize, respSize int
var reqContent string
switch _resp := resp.(type) {
case *pb.RangeResponse:
_req, ok := req.(*pb.RangeRequest)
if ok {
reqCount = 0
reqSize = _req.Size()
reqContent = _req.String()
}
if _resp != nil {
respCount = _resp.GetCount()
respSize = _resp.Size()
}
case *pb.PutResponse:
_req, ok := req.(*pb.PutRequest)
if ok {
reqCount = 1
reqSize = _req.Size()
reqContent = pb.NewLoggablePutRequest(_req).String()
// redact value field from request content, see PR #9821
}
if _resp != nil {
respCount = 0
respSize = _resp.Size()
}
case *pb.DeleteRangeResponse:
_req, ok := req.(*pb.DeleteRangeRequest)
if ok {
reqCount = 0
reqSize = _req.Size()
reqContent = _req.String()
}
if _resp != nil {
respCount = _resp.GetDeleted()
respSize = _resp.Size()
}
case *pb.TxnResponse:
_req, ok := req.(*pb.TxnRequest)
if ok && _resp != nil {
if _resp.GetSucceeded() { // determine the 'actual' count and size of request based on success or failure
reqCount = int64(len(_req.GetSuccess()))
reqSize = 0
for _, r := range _req.GetSuccess() {
reqSize += r.Size()
}
} else {
reqCount = int64(len(_req.GetFailure()))
reqSize = 0
for _, r := range _req.GetFailure() {
reqSize += r.Size()
}
}
reqContent = pb.NewLoggableTxnRequest(_req).String()
// redact value field from request content, see PR #9821
}
if _resp != nil {
respCount = 0
respSize = _resp.Size()
}
default:
reqCount = -1
reqSize = -1
respCount = -1
respSize = -1
}
logGenericRequestStats(lg, startTime, duration, remote, responseType, reqCount, reqSize, respCount, respSize, reqContent)
}
func logGenericRequestStats(lg *zap.Logger, startTime time.Time, duration time.Duration, remote string, responseType string,
reqCount int64, reqSize int, respCount int64, respSize int, reqContent string) {
if lg == nil {
plog.Debugf("start time = %v, "+
"time spent = %v, "+
"remote = %s, "+
"response type = %s, "+
"request count = %d, "+
"request size = %d, "+
"response count = %d, "+
"response size = %d, "+
"request content = %s",
startTime, duration, remote, responseType, reqCount, reqSize, respCount, respSize, reqContent,
)
} else {
lg.Debug("request stats",
zap.Time("start time", startTime),
zap.Duration("time spent", duration),
zap.String("remote", remote),
zap.String("response type", responseType),
zap.Int64("request count", reqCount),
zap.Int("request size", reqSize),
zap.Int64("response count", respCount),
zap.Int("response size", respSize),
zap.String("request content", reqContent),
)
} }
} }
...@@ -90,7 +209,7 @@ func newStreamInterceptor(s *etcdserver.EtcdServer) grpc.StreamServerInterceptor ...@@ -90,7 +209,7 @@ func newStreamInterceptor(s *etcdserver.EtcdServer) grpc.StreamServerInterceptor
} }
} }
return prometheus.StreamServerInterceptor(srv, ss, info, handler) return handler(srv, ss)
} }
} }
......
...@@ -59,7 +59,7 @@ func (as *InternalRaftStringer) String() string { ...@@ -59,7 +59,7 @@ func (as *InternalRaftStringer) String() string {
case as.Request.Put != nil: case as.Request.Put != nil:
return fmt.Sprintf("header:<%s> put:<%s>", return fmt.Sprintf("header:<%s> put:<%s>",
as.Request.Header.String(), as.Request.Header.String(),
newLoggablePutRequest(as.Request.Put).String(), NewLoggablePutRequest(as.Request.Put).String(),
) )
case as.Request.Txn != nil: case as.Request.Txn != nil:
return fmt.Sprintf("header:<%s> txn:<%s>", return fmt.Sprintf("header:<%s> txn:<%s>",
...@@ -121,7 +121,7 @@ func newLoggableRequestOp(op *RequestOp) *requestOpStringer { ...@@ -121,7 +121,7 @@ func newLoggableRequestOp(op *RequestOp) *requestOpStringer {
func (as *requestOpStringer) String() string { func (as *requestOpStringer) String() string {
switch op := as.Op.Request.(type) { switch op := as.Op.Request.(type) {
case *RequestOp_RequestPut: case *RequestOp_RequestPut:
return fmt.Sprintf("request_put:<%s>", newLoggablePutRequest(op.RequestPut).String()) return fmt.Sprintf("request_put:<%s>", NewLoggablePutRequest(op.RequestPut).String())
case *RequestOp_RequestTxn: case *RequestOp_RequestTxn:
return fmt.Sprintf("request_txn:<%s>", NewLoggableTxnRequest(op.RequestTxn).String()) return fmt.Sprintf("request_txn:<%s>", NewLoggableTxnRequest(op.RequestTxn).String())
default: default:
...@@ -167,7 +167,7 @@ type loggablePutRequest struct { ...@@ -167,7 +167,7 @@ type loggablePutRequest struct {
IgnoreLease bool `protobuf:"varint,6,opt,name=ignore_lease,proto3"` IgnoreLease bool `protobuf:"varint,6,opt,name=ignore_lease,proto3"`
} }
func newLoggablePutRequest(request *PutRequest) *loggablePutRequest { func NewLoggablePutRequest(request *PutRequest) *loggablePutRequest {
return &loggablePutRequest{ return &loggablePutRequest{
request.Key, request.Key,
len(request.Value), len(request.Value),
......
...@@ -90,6 +90,12 @@ var ( ...@@ -90,6 +90,12 @@ var (
Name: "slow_read_indexes_total", Name: "slow_read_indexes_total",
Help: "The total number of pending read indexes not in sync with leader's or timed out read index requests.", Help: "The total number of pending read indexes not in sync with leader's or timed out read index requests.",
}) })
readIndexFailed = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "etcd",
Subsystem: "server",
Name: "read_indexes_failed_total",
Help: "The total number of failed read indexes seen.",
})
quotaBackendBytes = prometheus.NewGauge(prometheus.GaugeOpts{ quotaBackendBytes = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "etcd", Namespace: "etcd",
Subsystem: "server", Subsystem: "server",
...@@ -110,6 +116,13 @@ var ( ...@@ -110,6 +116,13 @@ var (
Help: "Which Go version server is running with. 1 for 'server_go_version' label with current version.", Help: "Which Go version server is running with. 1 for 'server_go_version' label with current version.",
}, },
[]string{"server_go_version"}) []string{"server_go_version"})
serverID = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "etcd",
Subsystem: "server",
Name: "id",
Help: "Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID.",
},
[]string{"server_id"})
) )
func init() { func init() {
...@@ -124,9 +137,11 @@ func init() { ...@@ -124,9 +137,11 @@ func init() {
prometheus.MustRegister(proposalsFailed) prometheus.MustRegister(proposalsFailed)
prometheus.MustRegister(leaseExpired) prometheus.MustRegister(leaseExpired)
prometheus.MustRegister(slowReadIndex) prometheus.MustRegister(slowReadIndex)
prometheus.MustRegister(readIndexFailed)
prometheus.MustRegister(quotaBackendBytes) prometheus.MustRegister(quotaBackendBytes)
prometheus.MustRegister(currentVersion) prometheus.MustRegister(currentVersion)
prometheus.MustRegister(currentGoVersion) prometheus.MustRegister(currentGoVersion)
prometheus.MustRegister(serverID)
currentVersion.With(prometheus.Labels{ currentVersion.With(prometheus.Labels{
"server_version": version.Version, "server_version": version.Version,
......
...@@ -59,6 +59,7 @@ import ( ...@@ -59,6 +59,7 @@ import (
"github.com/coreos/go-semver/semver" "github.com/coreos/go-semver/semver"
"github.com/coreos/pkg/capnslog" "github.com/coreos/pkg/capnslog"
"github.com/prometheus/client_golang/prometheus"
) )
const ( const (
...@@ -435,6 +436,7 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) { ...@@ -435,6 +436,7 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
reqIDGen: idutil.NewGenerator(uint16(id), time.Now()), reqIDGen: idutil.NewGenerator(uint16(id), time.Now()),
forceVersionC: make(chan struct{}), forceVersionC: make(chan struct{}),
} }
serverID.With(prometheus.Labels{"server_id": id.String()}).Set(1)
srv.applyV2 = &applierV2store{store: srv.store, cluster: srv.cluster} srv.applyV2 = &applierV2store{store: srv.store, cluster: srv.cluster}
......
...@@ -634,6 +634,7 @@ func (s *EtcdServer) linearizableReadLoop() { ...@@ -634,6 +634,7 @@ func (s *EtcdServer) linearizableReadLoop() {
return return
} }
plog.Errorf("failed to get read index from raft: %v", err) plog.Errorf("failed to get read index from raft: %v", err)
readIndexFailed.Inc()
nr.notify(err) nr.notify(err)
continue continue
} }
...@@ -659,7 +660,7 @@ func (s *EtcdServer) linearizableReadLoop() { ...@@ -659,7 +660,7 @@ func (s *EtcdServer) linearizableReadLoop() {
} }
case <-time.After(s.Cfg.ReqTimeout()): case <-time.After(s.Cfg.ReqTimeout()):
plog.Warningf("timed out waiting for read index response") plog.Warningf("timed out waiting for read index response (local node might have slow network)")
nr.notify(ErrTimeout) nr.notify(ErrTimeout)
timeout = true timeout = true
slowReadIndex.Inc() slowReadIndex.Inc()
......
...@@ -22,6 +22,7 @@ import ( ...@@ -22,6 +22,7 @@ import (
"net/http" "net/http"
"path" "path"
"strings" "strings"
"time"
pioutil "github.com/coreos/etcd/pkg/ioutil" pioutil "github.com/coreos/etcd/pkg/ioutil"
"github.com/coreos/etcd/pkg/types" "github.com/coreos/etcd/pkg/types"
...@@ -149,6 +150,8 @@ func newSnapshotHandler(tr Transporter, r Raft, snapshotter *snap.Snapshotter, c ...@@ -149,6 +150,8 @@ func newSnapshotHandler(tr Transporter, r Raft, snapshotter *snap.Snapshotter, c
} }
} }
const unknownSnapshotSender = "UNKNOWN_SNAPSHOT_SENDER"
// ServeHTTP serves HTTP request to receive and process snapshot message. // ServeHTTP serves HTTP request to receive and process snapshot message.
// //
// If request sender dies without closing underlying TCP connection, // If request sender dies without closing underlying TCP connection,
...@@ -159,9 +162,12 @@ func newSnapshotHandler(tr Transporter, r Raft, snapshotter *snap.Snapshotter, c ...@@ -159,9 +162,12 @@ func newSnapshotHandler(tr Transporter, r Raft, snapshotter *snap.Snapshotter, c
// received and processed. // received and processed.
// 2. this case should happen rarely, so no further optimization is done. // 2. this case should happen rarely, so no further optimization is done.
func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
start := time.Now()
if r.Method != "POST" { if r.Method != "POST" {
w.Header().Set("Allow", "POST") w.Header().Set("Allow", "POST")
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
snapshotReceiveFailures.WithLabelValues(unknownSnapshotSender).Inc()
return return
} }
...@@ -169,6 +175,7 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ...@@ -169,6 +175,7 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if err := checkClusterCompatibilityFromHeader(r.Header, h.cid); err != nil { if err := checkClusterCompatibilityFromHeader(r.Header, h.cid); err != nil {
http.Error(w, err.Error(), http.StatusPreconditionFailed) http.Error(w, err.Error(), http.StatusPreconditionFailed)
snapshotReceiveFailures.WithLabelValues(unknownSnapshotSender).Inc()
return return
} }
...@@ -177,19 +184,22 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ...@@ -177,19 +184,22 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
dec := &messageDecoder{r: r.Body} dec := &messageDecoder{r: r.Body}
// let snapshots be very large since they can exceed 512MB for large installations // let snapshots be very large since they can exceed 512MB for large installations
m, err := dec.decodeLimit(uint64(1 << 63)) m, err := dec.decodeLimit(uint64(1 << 63))
from := types.ID(m.From).String()
if err != nil { if err != nil {
msg := fmt.Sprintf("failed to decode raft message (%v)", err) msg := fmt.Sprintf("failed to decode raft message (%v)", err)
plog.Errorf(msg) plog.Errorf(msg)
http.Error(w, msg, http.StatusBadRequest) http.Error(w, msg, http.StatusBadRequest)
recvFailures.WithLabelValues(r.RemoteAddr).Inc() recvFailures.WithLabelValues(r.RemoteAddr).Inc()
snapshotReceiveFailures.WithLabelValues(from).Inc()
return return
} }
receivedBytes.WithLabelValues(types.ID(m.From).String()).Add(float64(m.Size())) receivedBytes.WithLabelValues(from).Add(float64(m.Size()))
if m.Type != raftpb.MsgSnap { if m.Type != raftpb.MsgSnap {
plog.Errorf("unexpected raft message type %s on snapshot path", m.Type) plog.Errorf("unexpected raft message type %s on snapshot path", m.Type)
http.Error(w, "wrong raft message type", http.StatusBadRequest) http.Error(w, "wrong raft message type", http.StatusBadRequest)
snapshotReceiveFailures.WithLabelValues(from).Inc()
return return
} }
...@@ -200,9 +210,10 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ...@@ -200,9 +210,10 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
msg := fmt.Sprintf("failed to save KV snapshot (%v)", err) msg := fmt.Sprintf("failed to save KV snapshot (%v)", err)
plog.Error(msg) plog.Error(msg)
http.Error(w, msg, http.StatusInternalServerError) http.Error(w, msg, http.StatusInternalServerError)
snapshotReceiveFailures.WithLabelValues(from).Inc()
return return
} }
receivedBytes.WithLabelValues(types.ID(m.From).String()).Add(float64(n)) receivedBytes.WithLabelValues(from).Add(float64(n))
plog.Infof("received and saved database snapshot [index: %d, from: %s] successfully", m.Snapshot.Metadata.Index, types.ID(m.From)) plog.Infof("received and saved database snapshot [index: %d, from: %s] successfully", m.Snapshot.Metadata.Index, types.ID(m.From))
if err := h.r.Process(context.TODO(), m); err != nil { if err := h.r.Process(context.TODO(), m); err != nil {
...@@ -215,12 +226,16 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ...@@ -215,12 +226,16 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
msg := fmt.Sprintf("failed to process raft message (%v)", err) msg := fmt.Sprintf("failed to process raft message (%v)", err)
plog.Warningf(msg) plog.Warningf(msg)
http.Error(w, msg, http.StatusInternalServerError) http.Error(w, msg, http.StatusInternalServerError)
snapshotReceiveFailures.WithLabelValues(from).Inc()
} }
return return
} }
// Write StatusNoContent header after the message has been processed by // Write StatusNoContent header after the message has been processed by
// raft, which facilitates the client to report MsgSnap status. // raft, which facilitates the client to report MsgSnap status.
w.WriteHeader(http.StatusNoContent) w.WriteHeader(http.StatusNoContent)
snapshotReceive.WithLabelValues(from).Inc()
snapshotReceiveSeconds.WithLabelValues(from).Observe(time.Since(start).Seconds())
} }
type streamHandler struct { type streamHandler struct {
......
...@@ -53,6 +53,68 @@ var ( ...@@ -53,6 +53,68 @@ var (
[]string{"From"}, []string{"From"},
) )
snapshotSend = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "etcd",
Subsystem: "network",
Name: "snapshot_send_success",
Help: "Total number of successful snapshot sends",
},
[]string{"To"},
)
snapshotSendFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "etcd",
Subsystem: "network",
Name: "snapshot_send_failures",
Help: "Total number of snapshot send failures",
},
[]string{"To"},
)
snapshotSendSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "etcd",
Subsystem: "network",
Name: "snapshot_send_total_duration_seconds",
Help: "Total latency distributions of v3 snapshot sends",
// lowest bucket start of upper bound 0.1 sec (100 ms) with factor 2
// highest bucket start of 0.1 sec * 2^9 == 51.2 sec
Buckets: prometheus.ExponentialBuckets(0.1, 2, 10),
},
[]string{"To"},
)
snapshotReceive = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "etcd",
Subsystem: "network",
Name: "snapshot_receive_success",
Help: "Total number of successful snapshot receives",
},
[]string{"From"},
)
snapshotReceiveFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "etcd",
Subsystem: "network",
Name: "snapshot_receive_failures",
Help: "Total number of snapshot receive failures",
},
[]string{"From"},
)
snapshotReceiveSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "etcd",
Subsystem: "network",
Name: "snapshot_receive_total_duration_seconds",
Help: "Total latency distributions of v3 snapshot receives",
// lowest bucket start of upper bound 0.1 sec (100 ms) with factor 2
// highest bucket start of 0.1 sec * 2^9 == 51.2 sec
Buckets: prometheus.ExponentialBuckets(0.1, 2, 10),
},
[]string{"From"},
)
rtts = prometheus.NewHistogramVec(prometheus.HistogramOpts{ rtts = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "etcd", Namespace: "etcd",
Subsystem: "network", Subsystem: "network",
...@@ -69,5 +131,13 @@ func init() { ...@@ -69,5 +131,13 @@ func init() {
prometheus.MustRegister(receivedBytes) prometheus.MustRegister(receivedBytes)
prometheus.MustRegister(sentFailures) prometheus.MustRegister(sentFailures)
prometheus.MustRegister(recvFailures) prometheus.MustRegister(recvFailures)
prometheus.MustRegister(snapshotSend)
prometheus.MustRegister(snapshotSendFailures)
prometheus.MustRegister(snapshotSendSeconds)
prometheus.MustRegister(snapshotReceive)
prometheus.MustRegister(snapshotReceiveFailures)
prometheus.MustRegister(snapshotReceiveSeconds)
prometheus.MustRegister(rtts) prometheus.MustRegister(rtts)
} }
...@@ -56,7 +56,7 @@ func (s *peerStatus) deactivate(failure failureType, reason string) { ...@@ -56,7 +56,7 @@ func (s *peerStatus) deactivate(failure failureType, reason string) {
msg := fmt.Sprintf("failed to %s %s on %s (%s)", failure.action, s.id, failure.source, reason) msg := fmt.Sprintf("failed to %s %s on %s (%s)", failure.action, s.id, failure.source, reason)
if s.active { if s.active {
plog.Errorf(msg) plog.Errorf(msg)
plog.Infof("peer %s became inactive", s.id) plog.Infof("peer %s became inactive (message send to peer failed)", s.id)
s.active = false s.active = false
s.since = time.Time{} s.since = time.Time{}
return return
......
...@@ -17,6 +17,7 @@ package rafthttp ...@@ -17,6 +17,7 @@ package rafthttp
import ( import (
"time" "time"
"github.com/prometheus/client_golang/prometheus"
"github.com/xiang90/probing" "github.com/xiang90/probing"
) )
...@@ -28,7 +29,15 @@ var ( ...@@ -28,7 +29,15 @@ var (
statusErrorInterval = 5 * time.Second statusErrorInterval = 5 * time.Second
) )
func addPeerToProber(p probing.Prober, id string, us []string) { const (
// RoundTripperNameRaftMessage is the name of round-tripper that sends
// all other Raft messages, other than "snap.Message".
RoundTripperNameRaftMessage = "ROUND_TRIPPER_RAFT_MESSAGE"
// RoundTripperNameSnapshot is the name of round-tripper that sends merged snapshot message.
RoundTripperNameSnapshot = "ROUND_TRIPPER_SNAPSHOT"
)
func addPeerToProber(p probing.Prober, id string, us []string, roundTripperName string, rttSecProm *prometheus.HistogramVec) {
hus := make([]string, len(us)) hus := make([]string, len(us))
for i := range us { for i := range us {
hus[i] = us[i] + ProbingPrefix hus[i] = us[i] + ProbingPrefix
...@@ -40,26 +49,26 @@ func addPeerToProber(p probing.Prober, id string, us []string) { ...@@ -40,26 +49,26 @@ func addPeerToProber(p probing.Prober, id string, us []string) {
if err != nil { if err != nil {
plog.Errorf("failed to add peer %s into prober", id) plog.Errorf("failed to add peer %s into prober", id)
} else { } else {
go monitorProbingStatus(s, id) go monitorProbingStatus(s, id, roundTripperName, rttSecProm)
} }
} }
func monitorProbingStatus(s probing.Status, id string) { func monitorProbingStatus(s probing.Status, id string, roundTripperName string, rttSecProm *prometheus.HistogramVec) {
// set the first interval short to log error early. // set the first interval short to log error early.
interval := statusErrorInterval interval := statusErrorInterval
for { for {
select { select {
case <-time.After(interval): case <-time.After(interval):
if !s.Health() { if !s.Health() {
plog.Warningf("health check for peer %s could not connect: %v", id, s.Err()) plog.Warningf("health check for peer %s could not connect: %v (prober %q)", id, s.Err(), roundTripperName)
interval = statusErrorInterval interval = statusErrorInterval
} else { } else {
interval = statusMonitoringInterval interval = statusMonitoringInterval
} }
if s.ClockDiff() > time.Second { if s.ClockDiff() > time.Second {
plog.Warningf("the clock difference against peer %s is too high [%v > %v]", id, s.ClockDiff(), time.Second) plog.Warningf("the clock difference against peer %s is too high [%v > %v] (prober %q)", id, s.ClockDiff(), time.Second, roundTripperName)
} }
rtts.WithLabelValues(id).Observe(s.SRTT().Seconds()) rttSecProm.WithLabelValues(id).Observe(s.SRTT().Seconds())
case <-s.StopNotify(): case <-s.StopNotify():
return return
} }
......
...@@ -64,7 +64,10 @@ func newSnapshotSender(tr *Transport, picker *urlPicker, to types.ID, status *pe ...@@ -64,7 +64,10 @@ func newSnapshotSender(tr *Transport, picker *urlPicker, to types.ID, status *pe
func (s *snapshotSender) stop() { close(s.stopc) } func (s *snapshotSender) stop() { close(s.stopc) }
func (s *snapshotSender) send(merged snap.Message) { func (s *snapshotSender) send(merged snap.Message) {
start := time.Now()
m := merged.Message m := merged.Message
to := types.ID(m.To).String()
body := createSnapBody(merged) body := createSnapBody(merged)
defer body.Close() defer body.Close()
...@@ -92,14 +95,18 @@ func (s *snapshotSender) send(merged snap.Message) { ...@@ -92,14 +95,18 @@ func (s *snapshotSender) send(merged snap.Message) {
// machine knows about it, it would pause a while and retry sending // machine knows about it, it would pause a while and retry sending
// new snapshot message. // new snapshot message.
s.r.ReportSnapshot(m.To, raft.SnapshotFailure) s.r.ReportSnapshot(m.To, raft.SnapshotFailure)
sentFailures.WithLabelValues(types.ID(m.To).String()).Inc() sentFailures.WithLabelValues(to).Inc()
snapshotSendFailures.WithLabelValues(to).Inc()
return return
} }
s.status.activate() s.status.activate()
s.r.ReportSnapshot(m.To, raft.SnapshotFinish) s.r.ReportSnapshot(m.To, raft.SnapshotFinish)
plog.Infof("database snapshot [index: %d, to: %s] sent out successfully", m.Snapshot.Metadata.Index, types.ID(m.To)) plog.Infof("database snapshot [index: %d, to: %s] sent out successfully", m.Snapshot.Metadata.Index, types.ID(m.To))
sentBytes.WithLabelValues(types.ID(m.To).String()).Add(float64(merged.TotalSize)) sentBytes.WithLabelValues(to).Add(float64(merged.TotalSize))
snapshotSend.WithLabelValues(to).Inc()
snapshotSendSeconds.WithLabelValues(to).Observe(time.Since(start).Seconds())
} }
// post posts the given request. // post posts the given request.
......
...@@ -127,7 +127,8 @@ type Transport struct { ...@@ -127,7 +127,8 @@ type Transport struct {
remotes map[types.ID]*remote // remotes map that helps newly joined member to catch up remotes map[types.ID]*remote // remotes map that helps newly joined member to catch up
peers map[types.ID]Peer // peers map peers map[types.ID]Peer // peers map
prober probing.Prober pipelineProber probing.Prober
streamProber probing.Prober
} }
func (t *Transport) Start() error { func (t *Transport) Start() error {
...@@ -142,7 +143,8 @@ func (t *Transport) Start() error { ...@@ -142,7 +143,8 @@ func (t *Transport) Start() error {
} }
t.remotes = make(map[types.ID]*remote) t.remotes = make(map[types.ID]*remote)
t.peers = make(map[types.ID]Peer) t.peers = make(map[types.ID]Peer)
t.prober = probing.NewProber(t.pipelineRt) t.pipelineProber = probing.NewProber(t.pipelineRt)
t.streamProber = probing.NewProber(t.streamRt)
// If client didn't provide dial retry frequency, use the default // If client didn't provide dial retry frequency, use the default
// (100ms backoff between attempts to create a new stream), // (100ms backoff between attempts to create a new stream),
...@@ -210,7 +212,8 @@ func (t *Transport) Stop() { ...@@ -210,7 +212,8 @@ func (t *Transport) Stop() {
for _, p := range t.peers { for _, p := range t.peers {
p.stop() p.stop()
} }
t.prober.RemoveAll() t.pipelineProber.RemoveAll()
t.streamProber.RemoveAll()
if tr, ok := t.streamRt.(*http.Transport); ok { if tr, ok := t.streamRt.(*http.Transport); ok {
tr.CloseIdleConnections() tr.CloseIdleConnections()
} }
...@@ -289,8 +292,8 @@ func (t *Transport) AddPeer(id types.ID, us []string) { ...@@ -289,8 +292,8 @@ func (t *Transport) AddPeer(id types.ID, us []string) {
} }
fs := t.LeaderStats.Follower(id.String()) fs := t.LeaderStats.Follower(id.String())
t.peers[id] = startPeer(t, urls, id, fs) t.peers[id] = startPeer(t, urls, id, fs)
addPeerToProber(t.prober, id.String(), us) addPeerToProber(t.pipelineProber, id.String(), us, RoundTripperNameSnapshot, rtts)
addPeerToProber(t.streamProber, id.String(), us, RoundTripperNameRaftMessage, rtts)
plog.Infof("added peer %s", id) plog.Infof("added peer %s", id)
} }
...@@ -317,7 +320,8 @@ func (t *Transport) removePeer(id types.ID) { ...@@ -317,7 +320,8 @@ func (t *Transport) removePeer(id types.ID) {
} }
delete(t.peers, id) delete(t.peers, id)
delete(t.LeaderStats.Followers, id.String()) delete(t.LeaderStats.Followers, id.String())
t.prober.Remove(id.String()) t.pipelineProber.Remove(id.String())
t.streamProber.Remove(id.String())
plog.Infof("removed peer %s", id) plog.Infof("removed peer %s", id)
} }
...@@ -334,8 +338,10 @@ func (t *Transport) UpdatePeer(id types.ID, us []string) { ...@@ -334,8 +338,10 @@ func (t *Transport) UpdatePeer(id types.ID, us []string) {
} }
t.peers[id].update(urls) t.peers[id].update(urls)
t.prober.Remove(id.String()) t.pipelineProber.Remove(id.String())
addPeerToProber(t.prober, id.String(), us) addPeerToProber(t.pipelineProber, id.String(), us, RoundTripperNameSnapshot, rtts)
t.streamProber.Remove(id.String())
addPeerToProber(t.streamProber, id.String(), us, RoundTripperNameRaftMessage, rtts)
plog.Infof("updated peer %s", id) plog.Infof("updated peer %s", id)
} }
......
...@@ -21,6 +21,7 @@ import ( ...@@ -21,6 +21,7 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"time"
"github.com/coreos/etcd/pkg/fileutil" "github.com/coreos/etcd/pkg/fileutil"
) )
...@@ -30,6 +31,8 @@ var ErrNoDBSnapshot = errors.New("snap: snapshot file doesn't exist") ...@@ -30,6 +31,8 @@ var ErrNoDBSnapshot = errors.New("snap: snapshot file doesn't exist")
// SaveDBFrom saves snapshot of the database from the given reader. It // SaveDBFrom saves snapshot of the database from the given reader. It
// guarantees the save operation is atomic. // guarantees the save operation is atomic.
func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) {
start := time.Now()
f, err := ioutil.TempFile(s.dir, "tmp") f, err := ioutil.TempFile(s.dir, "tmp")
if err != nil { if err != nil {
return 0, err return 0, err
...@@ -37,7 +40,9 @@ func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { ...@@ -37,7 +40,9 @@ func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) {
var n int64 var n int64
n, err = io.Copy(f, r) n, err = io.Copy(f, r)
if err == nil { if err == nil {
fsyncStart := time.Now()
err = fileutil.Fsync(f) err = fileutil.Fsync(f)
snapDBFsyncSec.Observe(time.Since(fsyncStart).Seconds())
} }
f.Close() f.Close()
if err != nil { if err != nil {
...@@ -57,6 +62,7 @@ func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { ...@@ -57,6 +62,7 @@ func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) {
plog.Infof("saved database snapshot to disk [total bytes: %d]", n) plog.Infof("saved database snapshot to disk [total bytes: %d]", n)
snapDBSaveSec.Observe(time.Since(start).Seconds())
return n, nil return n, nil
} }
......
...@@ -33,9 +33,33 @@ var ( ...@@ -33,9 +33,33 @@ var (
Help: "The marshalling cost distributions of save called by snapshot.", Help: "The marshalling cost distributions of save called by snapshot.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), Buckets: prometheus.ExponentialBuckets(0.001, 2, 14),
}) })
snapDBSaveSec = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "etcd",
Subsystem: "snap_db",
Name: "save_total_duration_seconds",
Help: "The total latency distributions of v3 snapshot save",
// lowest bucket start of upper bound 0.1 sec (100 ms) with factor 2
// highest bucket start of 0.1 sec * 2^9 == 51.2 sec
Buckets: prometheus.ExponentialBuckets(0.1, 2, 10),
})
snapDBFsyncSec = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "etcd",
Subsystem: "snap_db",
Name: "fsync_duration_seconds",
Help: "The latency distributions of fsyncing .snap.db file",
// lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2
// highest bucket start of 0.001 sec * 2^13 == 8.192 sec
Buckets: prometheus.ExponentialBuckets(0.001, 2, 14),
})
) )
func init() { func init() {
prometheus.MustRegister(saveDurations) prometheus.MustRegister(saveDurations)
prometheus.MustRegister(marshallingDurations) prometheus.MustRegister(marshallingDurations)
prometheus.MustRegister(snapDBSaveSec)
prometheus.MustRegister(snapDBFsyncSec)
} }
...@@ -26,7 +26,7 @@ import ( ...@@ -26,7 +26,7 @@ import (
var ( var (
// MinClusterVersion is the min cluster version this etcd binary is compatible with. // MinClusterVersion is the min cluster version this etcd binary is compatible with.
MinClusterVersion = "3.0.0" MinClusterVersion = "3.0.0"
Version = "3.3.9" Version = "3.3.10"
APIVersion = "unknown" APIVersion = "unknown"
// Git SHA Value will be set during build // Git SHA Value will be set during build
......
CoreOS Project
Copyright 2018 CoreOS, Inc
This product includes software developed at CoreOS, Inc.
(http://www.coreos.com/).
// Copyright 2013-2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Semantic Versions http://semver.org
package semver package semver
import ( import (
"bytes" "bytes"
"errors" "errors"
"fmt" "fmt"
"regexp"
"strconv" "strconv"
"strings" "strings"
) )
...@@ -29,17 +45,45 @@ func splitOff(input *string, delim string) (val string) { ...@@ -29,17 +45,45 @@ func splitOff(input *string, delim string) (val string) {
return val return val
} }
func New(version string) *Version {
return Must(NewVersion(version))
}
func NewVersion(version string) (*Version, error) { func NewVersion(version string) (*Version, error) {
v := Version{} v := Version{}
if err := v.Set(version); err != nil {
return nil, err
}
return &v, nil
}
// Must is a helper for wrapping NewVersion and will panic if err is not nil.
func Must(v *Version, err error) *Version {
if err != nil {
panic(err)
}
return v
}
// Set parses and updates v from the given version string. Implements flag.Value
func (v *Version) Set(version string) error {
metadata := splitOff(&version, "+")
preRelease := PreRelease(splitOff(&version, "-"))
dotParts := strings.SplitN(version, ".", 3) dotParts := strings.SplitN(version, ".", 3)
if len(dotParts) != 3 { if len(dotParts) != 3 {
return nil, errors.New(fmt.Sprintf("%s is not in dotted-tri format", version)) return fmt.Errorf("%s is not in dotted-tri format", version)
}
if err := validateIdentifier(string(preRelease)); err != nil {
return fmt.Errorf("failed to validate pre-release: %v", err)
} }
v.Metadata = splitOff(&dotParts[2], "+") if err := validateIdentifier(metadata); err != nil {
v.PreRelease = PreRelease(splitOff(&dotParts[2], "-")) return fmt.Errorf("failed to validate metadata: %v", err)
}
parsed := make([]int64, 3, 3) parsed := make([]int64, 3, 3)
...@@ -47,63 +91,83 @@ func NewVersion(version string) (*Version, error) { ...@@ -47,63 +91,83 @@ func NewVersion(version string) (*Version, error) {
val, err := strconv.ParseInt(v, 10, 64) val, err := strconv.ParseInt(v, 10, 64)
parsed[i] = val parsed[i] = val
if err != nil { if err != nil {
return nil, err return err
} }
} }
v.Metadata = metadata
v.PreRelease = preRelease
v.Major = parsed[0] v.Major = parsed[0]
v.Minor = parsed[1] v.Minor = parsed[1]
v.Patch = parsed[2] v.Patch = parsed[2]
return nil
return &v, nil
} }
func Must(v *Version, err error) *Version { func (v Version) String() string {
if err != nil {
panic(err)
}
return v
}
func (v *Version) String() string {
var buffer bytes.Buffer var buffer bytes.Buffer
base := fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch) fmt.Fprintf(&buffer, "%d.%d.%d", v.Major, v.Minor, v.Patch)
buffer.WriteString(base)
if v.PreRelease != "" { if v.PreRelease != "" {
buffer.WriteString(fmt.Sprintf("-%s", v.PreRelease)) fmt.Fprintf(&buffer, "-%s", v.PreRelease)
} }
if v.Metadata != "" { if v.Metadata != "" {
buffer.WriteString(fmt.Sprintf("+%s", v.Metadata)) fmt.Fprintf(&buffer, "+%s", v.Metadata)
} }
return buffer.String() return buffer.String()
} }
func (v *Version) LessThan(versionB Version) bool { func (v *Version) UnmarshalYAML(unmarshal func(interface{}) error) error {
versionA := *v var data string
cmp := recursiveCompare(versionA.Slice(), versionB.Slice()) if err := unmarshal(&data); err != nil {
return err
}
return v.Set(data)
}
if cmp == 0 { func (v Version) MarshalJSON() ([]byte, error) {
cmp = preReleaseCompare(versionA, versionB) return []byte(`"` + v.String() + `"`), nil
}
func (v *Version) UnmarshalJSON(data []byte) error {
l := len(data)
if l == 0 || string(data) == `""` {
return nil
}
if l < 2 || data[0] != '"' || data[l-1] != '"' {
return errors.New("invalid semver string")
} }
return v.Set(string(data[1 : l-1]))
}
if cmp == -1 { // Compare tests if v is less than, equal to, or greater than versionB,
return true // returning -1, 0, or +1 respectively.
func (v Version) Compare(versionB Version) int {
if cmp := recursiveCompare(v.Slice(), versionB.Slice()); cmp != 0 {
return cmp
} }
return preReleaseCompare(v, versionB)
}
// Equal tests if v is equal to versionB.
func (v Version) Equal(versionB Version) bool {
return v.Compare(versionB) == 0
}
return false // LessThan tests if v is less than versionB.
func (v Version) LessThan(versionB Version) bool {
return v.Compare(versionB) < 0
} }
/* Slice converts the comparable parts of the semver into a slice of strings */ // Slice converts the comparable parts of the semver into a slice of integers.
func (v *Version) Slice() []int64 { func (v Version) Slice() []int64 {
return []int64{v.Major, v.Minor, v.Patch} return []int64{v.Major, v.Minor, v.Patch}
} }
func (p *PreRelease) Slice() []string { func (p PreRelease) Slice() []string {
preRelease := string(*p) preRelease := string(p)
return strings.Split(preRelease, ".") return strings.Split(preRelease, ".")
} }
...@@ -119,7 +183,7 @@ func preReleaseCompare(versionA Version, versionB Version) int { ...@@ -119,7 +183,7 @@ func preReleaseCompare(versionA Version, versionB Version) int {
return -1 return -1
} }
// If there is a prelease, check and compare each part. // If there is a prerelease, check and compare each part.
return recursivePreReleaseCompare(a.Slice(), b.Slice()) return recursivePreReleaseCompare(a.Slice(), b.Slice())
} }
...@@ -141,9 +205,12 @@ func recursiveCompare(versionA []int64, versionB []int64) int { ...@@ -141,9 +205,12 @@ func recursiveCompare(versionA []int64, versionB []int64) int {
} }
func recursivePreReleaseCompare(versionA []string, versionB []string) int { func recursivePreReleaseCompare(versionA []string, versionB []string) int {
// Handle slice length disparity. // A larger set of pre-release fields has a higher precedence than a smaller set,
// if all of the preceding identifiers are equal.
if len(versionA) == 0 { if len(versionA) == 0 {
// Nothing to compare too, so we return 0 if len(versionB) > 0 {
return -1
}
return 0 return 0
} else if len(versionB) == 0 { } else if len(versionB) == 0 {
// We're longer than versionB so return 1. // We're longer than versionB so return 1.
...@@ -153,7 +220,8 @@ func recursivePreReleaseCompare(versionA []string, versionB []string) int { ...@@ -153,7 +220,8 @@ func recursivePreReleaseCompare(versionA []string, versionB []string) int {
a := versionA[0] a := versionA[0]
b := versionB[0] b := versionB[0]
aInt := false; bInt := false aInt := false
bInt := false
aI, err := strconv.Atoi(versionA[0]) aI, err := strconv.Atoi(versionA[0])
if err == nil { if err == nil {
...@@ -165,6 +233,13 @@ func recursivePreReleaseCompare(versionA []string, versionB []string) int { ...@@ -165,6 +233,13 @@ func recursivePreReleaseCompare(versionA []string, versionB []string) int {
bInt = true bInt = true
} }
// Numeric identifiers always have lower precedence than non-numeric identifiers.
if aInt && !bInt {
return -1
} else if !aInt && bInt {
return 1
}
// Handle Integer Comparison // Handle Integer Comparison
if aInt && bInt { if aInt && bInt {
if aI > bI { if aI > bI {
...@@ -207,3 +282,15 @@ func (v *Version) BumpPatch() { ...@@ -207,3 +282,15 @@ func (v *Version) BumpPatch() {
v.PreRelease = PreRelease("") v.PreRelease = PreRelease("")
v.Metadata = "" v.Metadata = ""
} }
// validateIdentifier makes sure the provided identifier satisfies semver spec
func validateIdentifier(id string) error {
if id != "" && !reIdentifier.MatchString(id) {
return fmt.Errorf("%s is not a valid semver identifier", id)
}
return nil
}
// reIdentifier is a regular expression used to check that pre-release and metadata
// identifiers satisfy the spec requirements
var reIdentifier = regexp.MustCompile(`^[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*$`)
// Copyright 2013-2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package semver package semver
import ( import (
......
CoreOS Project
Copyright 2018 CoreOS, Inc
This product includes software developed at CoreOS, Inc.
(http://www.coreos.com/).
// Copyright 2014 Docker, Inc. // Copyright 2014 Docker, Inc.
// Copyright 2015-2018 CoreOS, Inc.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
...@@ -13,7 +14,11 @@ ...@@ -13,7 +14,11 @@
// limitations under the License. // limitations under the License.
// //
// Code forked from Docker project // Package daemon provides a Go implementation of the sd_notify protocol.
// It can be used to inform systemd of service start-up completion, watchdog
// events, and other status changes.
//
// https://www.freedesktop.org/software/systemd/man/sd_notify.html#Description
package daemon package daemon
import ( import (
...@@ -21,6 +26,25 @@ import ( ...@@ -21,6 +26,25 @@ import (
"os" "os"
) )
const (
// SdNotifyReady tells the service manager that service startup is finished
// or the service finished loading its configuration.
SdNotifyReady = "READY=1"
// SdNotifyStopping tells the service manager that the service is beginning
// its shutdown.
SdNotifyStopping = "STOPPING=1"
// SdNotifyReloading tells the service manager that this service is
// reloading its configuration. Note that you must call SdNotifyReady when
// it completed reloading.
SdNotifyReloading = "RELOADING=1"
// SdNotifyWatchdog tells the service manager to update the watchdog
// timestamp for the service.
SdNotifyWatchdog = "WATCHDOG=1"
)
// SdNotify sends a message to the init daemon. It is common to ignore the error. // SdNotify sends a message to the init daemon. It is common to ignore the error.
// If `unsetEnvironment` is true, the environment variable `NOTIFY_SOCKET` // If `unsetEnvironment` is true, the environment variable `NOTIFY_SOCKET`
// will be unconditionally unset. // will be unconditionally unset.
...@@ -29,7 +53,7 @@ import ( ...@@ -29,7 +53,7 @@ import (
// (false, nil) - notification not supported (i.e. NOTIFY_SOCKET is unset) // (false, nil) - notification not supported (i.e. NOTIFY_SOCKET is unset)
// (false, err) - notification supported, but failure happened (e.g. error connecting to NOTIFY_SOCKET or while sending data) // (false, err) - notification supported, but failure happened (e.g. error connecting to NOTIFY_SOCKET or while sending data)
// (true, nil) - notification supported, data has been sent // (true, nil) - notification supported, data has been sent
func SdNotify(unsetEnvironment bool, state string) (sent bool, err error) { func SdNotify(unsetEnvironment bool, state string) (bool, error) {
socketAddr := &net.UnixAddr{ socketAddr := &net.UnixAddr{
Name: os.Getenv("NOTIFY_SOCKET"), Name: os.Getenv("NOTIFY_SOCKET"),
Net: "unixgram", Net: "unixgram",
...@@ -41,10 +65,9 @@ func SdNotify(unsetEnvironment bool, state string) (sent bool, err error) { ...@@ -41,10 +65,9 @@ func SdNotify(unsetEnvironment bool, state string) (sent bool, err error) {
} }
if unsetEnvironment { if unsetEnvironment {
err = os.Unsetenv("NOTIFY_SOCKET") if err := os.Unsetenv("NOTIFY_SOCKET"); err != nil {
} return false, err
if err != nil { }
return false, err
} }
conn, err := net.DialUnix(socketAddr.Net, nil, socketAddr) conn, err := net.DialUnix(socketAddr.Net, nil, socketAddr)
...@@ -54,9 +77,7 @@ func SdNotify(unsetEnvironment bool, state string) (sent bool, err error) { ...@@ -54,9 +77,7 @@ func SdNotify(unsetEnvironment bool, state string) (sent bool, err error) {
} }
defer conn.Close() defer conn.Close()
_, err = conn.Write([]byte(state)) if _, err = conn.Write([]byte(state)); err != nil {
// Error sending the message
if err != nil {
return false, err return false, err
} }
return true, nil return true, nil
......
...@@ -21,10 +21,11 @@ import ( ...@@ -21,10 +21,11 @@ import (
"time" "time"
) )
// SdWatchdogEnabled return watchdog information for a service. // SdWatchdogEnabled returns watchdog information for a service.
// Process should send daemon.SdNotify("WATCHDOG=1") every time / 2. // Processes should call daemon.SdNotify(false, daemon.SdNotifyWatchdog) every
// If `unsetEnvironment` is true, the environment variables `WATCHDOG_USEC` // time / 2.
// and `WATCHDOG_PID` will be unconditionally unset. // If `unsetEnvironment` is true, the environment variables `WATCHDOG_USEC` and
// `WATCHDOG_PID` will be unconditionally unset.
// //
// It returns one of the following: // It returns one of the following:
// (0, nil) - watchdog isn't enabled or we aren't the watched PID. // (0, nil) - watchdog isn't enabled or we aren't the watched PID.
......
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
package dbus package dbus
import ( import (
"encoding/hex"
"fmt" "fmt"
"os" "os"
"strconv" "strconv"
...@@ -60,6 +61,27 @@ func PathBusEscape(path string) string { ...@@ -60,6 +61,27 @@ func PathBusEscape(path string) string {
return string(n) return string(n)
} }
// pathBusUnescape is the inverse of PathBusEscape.
func pathBusUnescape(path string) string {
if path == "_" {
return ""
}
n := []byte{}
for i := 0; i < len(path); i++ {
c := path[i]
if c == '_' && i+2 < len(path) {
res, err := hex.DecodeString(path[i+1 : i+3])
if err == nil {
n = append(n, res...)
}
i += 2
} else {
n = append(n, c)
}
}
return string(n)
}
// Conn is a connection to systemd's dbus endpoint. // Conn is a connection to systemd's dbus endpoint.
type Conn struct { type Conn struct {
// sysconn/sysobj are only used to call dbus methods // sysconn/sysobj are only used to call dbus methods
...@@ -74,13 +96,18 @@ type Conn struct { ...@@ -74,13 +96,18 @@ type Conn struct {
jobs map[dbus.ObjectPath]chan<- string jobs map[dbus.ObjectPath]chan<- string
sync.Mutex sync.Mutex
} }
subscriber struct { subStateSubscriber struct {
updateCh chan<- *SubStateUpdate updateCh chan<- *SubStateUpdate
errCh chan<- error errCh chan<- error
sync.Mutex sync.Mutex
ignore map[dbus.ObjectPath]int64 ignore map[dbus.ObjectPath]int64
cleanIgnore int64 cleanIgnore int64
} }
propertiesSubscriber struct {
updateCh chan<- *PropertiesUpdate
errCh chan<- error
sync.Mutex
}
} }
// New establishes a connection to any available bus and authenticates. // New establishes a connection to any available bus and authenticates.
...@@ -152,7 +179,7 @@ func NewConnection(dialBus func() (*dbus.Conn, error)) (*Conn, error) { ...@@ -152,7 +179,7 @@ func NewConnection(dialBus func() (*dbus.Conn, error)) (*Conn, error) {
sigobj: systemdObject(sigconn), sigobj: systemdObject(sigconn),
} }
c.subscriber.ignore = make(map[dbus.ObjectPath]int64) c.subStateSubscriber.ignore = make(map[dbus.ObjectPath]int64)
c.jobListener.jobs = make(map[dbus.ObjectPath]chan<- string) c.jobListener.jobs = make(map[dbus.ObjectPath]chan<- string)
// Setup the listeners on jobs so that we can get completions // Setup the listeners on jobs so that we can get completions
......
// Copyright 2015 CoreOS, Inc. // Copyright 2015, 2018 CoreOS, Inc.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
...@@ -16,6 +16,7 @@ package dbus ...@@ -16,6 +16,7 @@ package dbus
import ( import (
"errors" "errors"
"fmt"
"path" "path"
"strconv" "strconv"
...@@ -148,14 +149,27 @@ func (c *Conn) ResetFailedUnit(name string) error { ...@@ -148,14 +149,27 @@ func (c *Conn) ResetFailedUnit(name string) error {
return c.sysobj.Call("org.freedesktop.systemd1.Manager.ResetFailedUnit", 0, name).Store() return c.sysobj.Call("org.freedesktop.systemd1.Manager.ResetFailedUnit", 0, name).Store()
} }
// getProperties takes the unit name and returns all of its dbus object properties, for the given dbus interface // SystemState returns the systemd state. Equivalent to `systemctl is-system-running`.
func (c *Conn) getProperties(unit string, dbusInterface string) (map[string]interface{}, error) { func (c *Conn) SystemState() (*Property, error) {
var err error
var prop dbus.Variant
obj := c.sysconn.Object("org.freedesktop.systemd1", "/org/freedesktop/systemd1")
err = obj.Call("org.freedesktop.DBus.Properties.Get", 0, "org.freedesktop.systemd1.Manager", "SystemState").Store(&prop)
if err != nil {
return nil, err
}
return &Property{Name: "SystemState", Value: prop}, nil
}
// getProperties takes the unit path and returns all of its dbus object properties, for the given dbus interface
func (c *Conn) getProperties(path dbus.ObjectPath, dbusInterface string) (map[string]interface{}, error) {
var err error var err error
var props map[string]dbus.Variant var props map[string]dbus.Variant
path := unitPath(unit)
if !path.IsValid() { if !path.IsValid() {
return nil, errors.New("invalid unit name: " + unit) return nil, fmt.Errorf("invalid unit name: %v", path)
} }
obj := c.sysconn.Object("org.freedesktop.systemd1", path) obj := c.sysconn.Object("org.freedesktop.systemd1", path)
...@@ -172,9 +186,15 @@ func (c *Conn) getProperties(unit string, dbusInterface string) (map[string]inte ...@@ -172,9 +186,15 @@ func (c *Conn) getProperties(unit string, dbusInterface string) (map[string]inte
return out, nil return out, nil
} }
// GetUnitProperties takes the unit name and returns all of its dbus object properties. // GetUnitProperties takes the (unescaped) unit name and returns all of its dbus object properties.
func (c *Conn) GetUnitProperties(unit string) (map[string]interface{}, error) { func (c *Conn) GetUnitProperties(unit string) (map[string]interface{}, error) {
return c.getProperties(unit, "org.freedesktop.systemd1.Unit") path := unitPath(unit)
return c.getProperties(path, "org.freedesktop.systemd1.Unit")
}
// GetUnitProperties takes the (escaped) unit path and returns all of its dbus object properties.
func (c *Conn) GetUnitPathProperties(path dbus.ObjectPath) (map[string]interface{}, error) {
return c.getProperties(path, "org.freedesktop.systemd1.Unit")
} }
func (c *Conn) getProperty(unit string, dbusInterface string, propertyName string) (*Property, error) { func (c *Conn) getProperty(unit string, dbusInterface string, propertyName string) (*Property, error) {
...@@ -208,7 +228,8 @@ func (c *Conn) GetServiceProperty(service string, propertyName string) (*Propert ...@@ -208,7 +228,8 @@ func (c *Conn) GetServiceProperty(service string, propertyName string) (*Propert
// Valid values for unitType: Service, Socket, Target, Device, Mount, Automount, Snapshot, Timer, Swap, Path, Slice, Scope // Valid values for unitType: Service, Socket, Target, Device, Mount, Automount, Snapshot, Timer, Swap, Path, Slice, Scope
// return "dbus.Error: Unknown interface" if the unitType is not the correct type of the unit // return "dbus.Error: Unknown interface" if the unitType is not the correct type of the unit
func (c *Conn) GetUnitTypeProperties(unit string, unitType string) (map[string]interface{}, error) { func (c *Conn) GetUnitTypeProperties(unit string, unitType string) (map[string]interface{}, error) {
return c.getProperties(unit, "org.freedesktop.systemd1."+unitType) path := unitPath(unit)
return c.getProperties(path, "org.freedesktop.systemd1."+unitType)
} }
// SetUnitProperties() may be used to modify certain unit properties at runtime. // SetUnitProperties() may be used to modify certain unit properties at runtime.
...@@ -292,6 +313,7 @@ func (c *Conn) ListUnitsByPatterns(states []string, patterns []string) ([]UnitSt ...@@ -292,6 +313,7 @@ func (c *Conn) ListUnitsByPatterns(states []string, patterns []string) ([]UnitSt
// names and returns an UnitStatus array. Comparing to ListUnitsByPatterns // names and returns an UnitStatus array. Comparing to ListUnitsByPatterns
// method, this method returns statuses even for inactive or non-existing // method, this method returns statuses even for inactive or non-existing
// units. Input array should contain exact unit names, but not patterns. // units. Input array should contain exact unit names, but not patterns.
// Note: Requires systemd v230 or higher
func (c *Conn) ListUnitsByNames(units []string) ([]UnitStatus, error) { func (c *Conn) ListUnitsByNames(units []string) ([]UnitStatus, error) {
return c.listUnitsInternal(c.sysobj.Call("org.freedesktop.systemd1.Manager.ListUnitsByNames", 0, units).Store) return c.listUnitsInternal(c.sysobj.Call("org.freedesktop.systemd1.Manager.ListUnitsByNames", 0, units).Store)
} }
...@@ -563,3 +585,8 @@ func (c *Conn) Reload() error { ...@@ -563,3 +585,8 @@ func (c *Conn) Reload() error {
func unitPath(name string) dbus.ObjectPath { func unitPath(name string) dbus.ObjectPath {
return dbus.ObjectPath("/org/freedesktop/systemd1/unit/" + PathBusEscape(name)) return dbus.ObjectPath("/org/freedesktop/systemd1/unit/" + PathBusEscape(name))
} }
// unitName returns the unescaped base element of the supplied escaped path
func unitName(dpath dbus.ObjectPath) string {
return pathBusUnescape(path.Base(string(dpath)))
}
...@@ -36,7 +36,7 @@ func (s *set) Length() int { ...@@ -36,7 +36,7 @@ func (s *set) Length() int {
} }
func (s *set) Values() (values []string) { func (s *set) Values() (values []string) {
for val, _ := range s.data { for val := range s.data {
values = append(values, val) values = append(values, val)
} }
return return
......
...@@ -16,6 +16,7 @@ package dbus ...@@ -16,6 +16,7 @@ package dbus
import ( import (
"errors" "errors"
"log"
"time" "time"
"github.com/godbus/dbus" "github.com/godbus/dbus"
...@@ -36,22 +37,12 @@ func (c *Conn) Subscribe() error { ...@@ -36,22 +37,12 @@ func (c *Conn) Subscribe() error {
c.sigconn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0, c.sigconn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0,
"type='signal',interface='org.freedesktop.DBus.Properties',member='PropertiesChanged'") "type='signal',interface='org.freedesktop.DBus.Properties',member='PropertiesChanged'")
err := c.sigobj.Call("org.freedesktop.systemd1.Manager.Subscribe", 0).Store() return c.sigobj.Call("org.freedesktop.systemd1.Manager.Subscribe", 0).Store()
if err != nil {
return err
}
return nil
} }
// Unsubscribe this connection from systemd dbus events. // Unsubscribe this connection from systemd dbus events.
func (c *Conn) Unsubscribe() error { func (c *Conn) Unsubscribe() error {
err := c.sigobj.Call("org.freedesktop.systemd1.Manager.Unsubscribe", 0).Store() return c.sigobj.Call("org.freedesktop.systemd1.Manager.Unsubscribe", 0).Store()
if err != nil {
return err
}
return nil
} }
func (c *Conn) dispatch() { func (c *Conn) dispatch() {
...@@ -70,7 +61,8 @@ func (c *Conn) dispatch() { ...@@ -70,7 +61,8 @@ func (c *Conn) dispatch() {
c.jobComplete(signal) c.jobComplete(signal)
} }
if c.subscriber.updateCh == nil { if c.subStateSubscriber.updateCh == nil &&
c.propertiesSubscriber.updateCh == nil {
continue continue
} }
...@@ -84,6 +76,12 @@ func (c *Conn) dispatch() { ...@@ -84,6 +76,12 @@ func (c *Conn) dispatch() {
case "org.freedesktop.DBus.Properties.PropertiesChanged": case "org.freedesktop.DBus.Properties.PropertiesChanged":
if signal.Body[0].(string) == "org.freedesktop.systemd1.Unit" { if signal.Body[0].(string) == "org.freedesktop.systemd1.Unit" {
unitPath = signal.Path unitPath = signal.Path
if len(signal.Body) >= 2 {
if changed, ok := signal.Body[1].(map[string]dbus.Variant); ok {
c.sendPropertiesUpdate(unitPath, changed)
}
}
} }
} }
...@@ -169,42 +167,80 @@ type SubStateUpdate struct { ...@@ -169,42 +167,80 @@ type SubStateUpdate struct {
// is full, it attempts to write an error to errCh; if errCh is full, the error // is full, it attempts to write an error to errCh; if errCh is full, the error
// passes silently. // passes silently.
func (c *Conn) SetSubStateSubscriber(updateCh chan<- *SubStateUpdate, errCh chan<- error) { func (c *Conn) SetSubStateSubscriber(updateCh chan<- *SubStateUpdate, errCh chan<- error) {
c.subscriber.Lock() if c == nil {
defer c.subscriber.Unlock() msg := "nil receiver"
c.subscriber.updateCh = updateCh select {
c.subscriber.errCh = errCh case errCh <- errors.New(msg):
default:
log.Printf("full error channel while reporting: %s\n", msg)
}
return
}
c.subStateSubscriber.Lock()
defer c.subStateSubscriber.Unlock()
c.subStateSubscriber.updateCh = updateCh
c.subStateSubscriber.errCh = errCh
} }
func (c *Conn) sendSubStateUpdate(path dbus.ObjectPath) { func (c *Conn) sendSubStateUpdate(unitPath dbus.ObjectPath) {
c.subscriber.Lock() c.subStateSubscriber.Lock()
defer c.subscriber.Unlock() defer c.subStateSubscriber.Unlock()
if c.subStateSubscriber.updateCh == nil {
return
}
if c.shouldIgnore(path) { isIgnored := c.shouldIgnore(unitPath)
defer c.cleanIgnore()
if isIgnored {
return return
} }
info, err := c.GetUnitProperties(string(path)) info, err := c.GetUnitPathProperties(unitPath)
if err != nil { if err != nil {
select { select {
case c.subscriber.errCh <- err: case c.subStateSubscriber.errCh <- err:
default: default:
log.Printf("full error channel while reporting: %s\n", err)
} }
return
} }
defer c.updateIgnore(unitPath, info)
name := info["Id"].(string) name, ok := info["Id"].(string)
substate := info["SubState"].(string) if !ok {
msg := "failed to cast info.Id"
select {
case c.subStateSubscriber.errCh <- errors.New(msg):
default:
log.Printf("full error channel while reporting: %s\n", err)
}
return
}
substate, ok := info["SubState"].(string)
if !ok {
msg := "failed to cast info.SubState"
select {
case c.subStateSubscriber.errCh <- errors.New(msg):
default:
log.Printf("full error channel while reporting: %s\n", msg)
}
return
}
update := &SubStateUpdate{name, substate} update := &SubStateUpdate{name, substate}
select { select {
case c.subscriber.updateCh <- update: case c.subStateSubscriber.updateCh <- update:
default: default:
msg := "update channel is full"
select { select {
case c.subscriber.errCh <- errors.New("update channel full!"): case c.subStateSubscriber.errCh <- errors.New(msg):
default: default:
log.Printf("full error channel while reporting: %s\n", msg)
} }
return
} }
c.updateIgnore(path, info)
} }
// The ignore functions work around a wart in the systemd dbus interface. // The ignore functions work around a wart in the systemd dbus interface.
...@@ -222,29 +258,76 @@ func (c *Conn) sendSubStateUpdate(path dbus.ObjectPath) { ...@@ -222,29 +258,76 @@ func (c *Conn) sendSubStateUpdate(path dbus.ObjectPath) {
// the properties). // the properties).
func (c *Conn) shouldIgnore(path dbus.ObjectPath) bool { func (c *Conn) shouldIgnore(path dbus.ObjectPath) bool {
t, ok := c.subscriber.ignore[path] t, ok := c.subStateSubscriber.ignore[path]
return ok && t >= time.Now().UnixNano() return ok && t >= time.Now().UnixNano()
} }
func (c *Conn) updateIgnore(path dbus.ObjectPath, info map[string]interface{}) { func (c *Conn) updateIgnore(path dbus.ObjectPath, info map[string]interface{}) {
c.cleanIgnore() loadState, ok := info["LoadState"].(string)
if !ok {
return
}
// unit is unloaded - it will trigger bad systemd dbus behavior // unit is unloaded - it will trigger bad systemd dbus behavior
if info["LoadState"].(string) == "not-found" { if loadState == "not-found" {
c.subscriber.ignore[path] = time.Now().UnixNano() + ignoreInterval c.subStateSubscriber.ignore[path] = time.Now().UnixNano() + ignoreInterval
} }
} }
// without this, ignore would grow unboundedly over time // without this, ignore would grow unboundedly over time
func (c *Conn) cleanIgnore() { func (c *Conn) cleanIgnore() {
now := time.Now().UnixNano() now := time.Now().UnixNano()
if c.subscriber.cleanIgnore < now { if c.subStateSubscriber.cleanIgnore < now {
c.subscriber.cleanIgnore = now + cleanIgnoreInterval c.subStateSubscriber.cleanIgnore = now + cleanIgnoreInterval
for p, t := range c.subscriber.ignore { for p, t := range c.subStateSubscriber.ignore {
if t < now { if t < now {
delete(c.subscriber.ignore, p) delete(c.subStateSubscriber.ignore, p)
} }
} }
} }
} }
// PropertiesUpdate holds a map of a unit's changed properties
type PropertiesUpdate struct {
UnitName string
Changed map[string]dbus.Variant
}
// SetPropertiesSubscriber writes to updateCh when any unit's properties
// change. Every property change reported by systemd will be sent; that is, no
// transitions will be "missed" (as they might be with SetSubStateSubscriber).
// However, state changes will only be written to the channel with non-blocking
// writes. If updateCh is full, it attempts to write an error to errCh; if
// errCh is full, the error passes silently.
func (c *Conn) SetPropertiesSubscriber(updateCh chan<- *PropertiesUpdate, errCh chan<- error) {
c.propertiesSubscriber.Lock()
defer c.propertiesSubscriber.Unlock()
c.propertiesSubscriber.updateCh = updateCh
c.propertiesSubscriber.errCh = errCh
}
// we don't need to worry about shouldIgnore() here because
// sendPropertiesUpdate doesn't call GetProperties()
func (c *Conn) sendPropertiesUpdate(unitPath dbus.ObjectPath, changedProps map[string]dbus.Variant) {
c.propertiesSubscriber.Lock()
defer c.propertiesSubscriber.Unlock()
if c.propertiesSubscriber.updateCh == nil {
return
}
update := &PropertiesUpdate{unitName(unitPath), changedProps}
select {
case c.propertiesSubscriber.updateCh <- update:
default:
msg := "update channel is full"
select {
case c.propertiesSubscriber.errCh <- errors.New(msg):
default:
log.Printf("full error channel while reporting: %s\n", msg)
}
return
}
}
...@@ -103,7 +103,10 @@ func Send(message string, priority Priority, vars map[string]string) error { ...@@ -103,7 +103,10 @@ func Send(message string, priority Priority, vars map[string]string) error {
if !ok { if !ok {
return journalError("can't send file through non-Unix connection") return journalError("can't send file through non-Unix connection")
} }
unixConn.WriteMsgUnix([]byte{}, rights, nil) _, _, err = unixConn.WriteMsgUnix([]byte{}, rights, nil)
if err != nil {
return journalError(err.Error())
}
} else if err != nil { } else if err != nil {
return journalError(err.Error()) return journalError(err.Error())
} }
...@@ -165,7 +168,7 @@ func tempFd() (*os.File, error) { ...@@ -165,7 +168,7 @@ func tempFd() (*os.File, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
syscall.Unlink(file.Name()) err = syscall.Unlink(file.Name())
if err != nil { if err != nil {
return nil, err return nil, err
} }
......
...@@ -122,11 +122,12 @@ func runningFromSystemService() (ret bool, err error) { ...@@ -122,11 +122,12 @@ func runningFromSystemService() (ret bool, err error) {
errno := C.my_sd_pid_get_owner_uid(sd_pid_get_owner_uid, 0, &uid) errno := C.my_sd_pid_get_owner_uid(sd_pid_get_owner_uid, 0, &uid)
serrno := syscall.Errno(-errno) serrno := syscall.Errno(-errno)
// when we're running from a unit file, sd_pid_get_owner_uid returns // when we're running from a unit file, sd_pid_get_owner_uid returns
// ENOENT (systemd <220) or ENXIO (systemd >=220) // ENOENT (systemd <220), ENXIO (systemd 220-223), or ENODATA
// (systemd >=234)
switch { switch {
case errno >= 0: case errno >= 0:
ret = false ret = false
case serrno == syscall.ENOENT, serrno == syscall.ENXIO: case serrno == syscall.ENOENT, serrno == syscall.ENXIO, serrno == syscall.ENODATA:
// Since the implementation of sessions in systemd relies on // Since the implementation of sessions in systemd relies on
// the `pam_systemd` module, using the sd_pid_get_owner_uid // the `pam_systemd` module, using the sd_pid_get_owner_uid
// heuristic alone can result in false positives if that module // heuristic alone can result in false positives if that module
......
...@@ -21,7 +21,7 @@ Still the job of `main` to expose these configurations. `main` may delegate this ...@@ -21,7 +21,7 @@ Still the job of `main` to expose these configurations. `main` may delegate this
Splitting streams is probably not the job of your program, but rather, your log aggregation framework. If you must split output streams, again, `main` configures this and you can write a very simple two-output struct that satisfies io.Writer. Splitting streams is probably not the job of your program, but rather, your log aggregation framework. If you must split output streams, again, `main` configures this and you can write a very simple two-output struct that satisfies io.Writer.
Fancy colorful formatting and JSON output are beyond the scope of a basic logging framework -- they're application/log-collector dependant. These are, at best, provided as options, but more likely, provided by your application. Fancy colorful formatting and JSON output are beyond the scope of a basic logging framework -- they're application/log-collector dependent. These are, at best, provided as options, but more likely, provided by your application.
##### Log objects are an interface ##### Log objects are an interface
......
...@@ -95,6 +95,11 @@ func (l *LogLevel) Set(s string) error { ...@@ -95,6 +95,11 @@ func (l *LogLevel) Set(s string) error {
return nil return nil
} }
// Returns an empty string, only here to fulfill the pflag.Value interface.
func (l *LogLevel) Type() string {
return ""
}
// ParseLevel translates some potential loglevel strings into their corresponding levels. // ParseLevel translates some potential loglevel strings into their corresponding levels.
func ParseLevel(s string) (LogLevel, error) { func ParseLevel(s string) (LogLevel, error) {
switch s { switch s {
......
...@@ -37,6 +37,14 @@ func (p *PackageLogger) internalLog(depth int, inLevel LogLevel, entries ...inte ...@@ -37,6 +37,14 @@ func (p *PackageLogger) internalLog(depth int, inLevel LogLevel, entries ...inte
} }
} }
// SetLevel allows users to change the current logging level.
func (p *PackageLogger) SetLevel(l LogLevel) {
logger.Lock()
defer logger.Unlock()
p.level = l
}
// LevelAt checks if the given log level will be outputted under current setting.
func (p *PackageLogger) LevelAt(l LogLevel) bool { func (p *PackageLogger) LevelAt(l LogLevel) bool {
logger.Lock() logger.Lock()
defer logger.Unlock() defer logger.Unlock()
...@@ -81,6 +89,12 @@ func (p *PackageLogger) Panic(args ...interface{}) { ...@@ -81,6 +89,12 @@ func (p *PackageLogger) Panic(args ...interface{}) {
panic(s) panic(s)
} }
func (p *PackageLogger) Panicln(args ...interface{}) {
s := fmt.Sprintln(args...)
p.internalLog(calldepth, CRITICAL, s)
panic(s)
}
func (p *PackageLogger) Fatalf(format string, args ...interface{}) { func (p *PackageLogger) Fatalf(format string, args ...interface{}) {
p.Logf(CRITICAL, format, args...) p.Logf(CRITICAL, format, args...)
os.Exit(1) os.Exit(1)
......
# Created by .ignore support plugin (hsz.mobi)
### Go template
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
### Windows template
# Windows image file caches
Thumbs.db
ehthumbs.db
# Folder config file
Desktop.ini
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Windows Installer files
*.cab
*.msi
*.msm
*.msp
# Windows shortcuts
*.lnk
### Kate template
# Swap Files #
.*.kate-swp
.swp.*
### SublimeText template
# cache files for sublime text
*.tmlanguage.cache
*.tmPreferences.cache
*.stTheme.cache
# workspace files are user-specific
*.sublime-workspace
# project files should be checked into the repository, unless a significant
# proportion of contributors will probably not be using SublimeText
# *.sublime-project
# sftp configuration file
sftp-config.json
### Linux template
*~
# temporary files which can be created if a process still has a handle open of a deleted file
.fuse_hidden*
# KDE directory preferences
.directory
# Linux trash folder which might appear on any partition or disk
.Trash-*
### JetBrains template
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff:
.idea
.idea/tasks.xml
.idea/dictionaries
.idea/vcs.xml
.idea/jsLibraryMappings.xml
# Sensitive or high-churn files:
.idea/dataSources.ids
.idea/dataSources.xml
.idea/dataSources.local.xml
.idea/sqlDataSources.xml
.idea/dynamic.xml
.idea/uiDesigner.xml
# Gradle:
.idea/gradle.xml
.idea/libraries
# Mongo Explorer plugin:
.idea/mongoSettings.xml
## File-based project format:
*.iws
## Plugin-specific files:
# IntelliJ
/out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
### Xcode template
# Xcode
#
# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore
## Build generated
build/
DerivedData/
## Various settings
*.pbxuser
!default.pbxuser
*.mode1v3
!default.mode1v3
*.mode2v3
!default.mode2v3
*.perspectivev3
!default.perspectivev3
xcuserdata/
## Other
*.moved-aside
*.xccheckout
*.xcscmblueprint
### Eclipse template
.metadata
bin/
tmp/
*.tmp
*.bak
*.swp
*~.nib
local.properties
.settings/
.loadpath
.recommenders
# Eclipse Core
.project
# External tool builders
.externalToolBuilders/
# Locally stored "Eclipse launch configurations"
*.launch
# PyDev specific (Python IDE for Eclipse)
*.pydevproject
# CDT-specific (C/C++ Development Tooling)
.cproject
# JDT-specific (Eclipse Java Development Tools)
.classpath
# Java annotation processor (APT)
.factorypath
# PDT-specific (PHP Development Tools)
.buildpath
# sbteclipse plugin
.target
# Tern plugin
.tern-project
# TeXlipse plugin
.texlipse
# STS (Spring Tool Suite)
.springBeans
# Code Recommenders
.recommenders/
coverage.txt
#vendor
vendor/
sudo: false
language: go
go:
- 1.8.x
env:
- DEP_VERSION="0.3.2"
before_install:
# Download the binary to bin folder in $GOPATH
- curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep
# Make the binary executable
- chmod +x $GOPATH/bin/dep
install:
- dep ensure
script:
- make checkdocs
- make test
after_success:
- bash <(curl -s https://codecov.io/bash)
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"chain.go",
"doc.go",
"wrappers.go",
],
importmap = "k8s.io/kubernetes/vendor/github.com/grpc-ecosystem/go-grpc-middleware",
importpath = "github.com/grpc-ecosystem/go-grpc-middleware",
visibility = ["//visibility:public"],
deps = [
"//vendor/golang.org/x/net/context:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
Types of changes:
- `Added` for new features.
- `Changed` for changes in existing functionality.
- `Deprecated` for soon-to-be removed features.
- `Removed` for now removed features.
- `Fixed` for any bug fixes.
- `Security` in case of vulnerabilities.
## [Unreleased]
### Added
- This CHANGELOG file to keep track of changes.
## 1.0.0 - 2018-05-08
### Added
- grpc_auth
- grpc_ctxtags
- grpc_zap
- grpc_logrus
- grpc_opentracing
- grpc_retry
- grpc_validator
- grpc_recovery
[Unreleased]: https://github.com/grpc-ecosystem/go-grpc-middleware/compare/v1.0.0...HEAD
# Contributing
We would love to have people submit pull requests and help make `grpc-ecosystem/go-grpc-middleware` even better 👍.
Fork, then clone the repo:
```bash
git clone git@github.com:your-username/go-grpc-middleware.git
```
Before checking in please run the following:
```bash
make all
```
This will `vet`, `fmt`, regenerate documentation and run all tests.
Push to your fork and open a pull request.
\ No newline at end of file
# grpc_middleware
`import "github.com/grpc-ecosystem/go-grpc-middleware"`
* [Overview](#pkg-overview)
* [Imported Packages](#pkg-imports)
* [Index](#pkg-index)
## <a name="pkg-overview">Overview</a>
`grpc_middleware` is a collection of gRPC middleware packages: interceptors, helpers and tools.
### Middleware
gRPC is a fantastic RPC middleware, which sees a lot of adoption in the Golang world. However, the
upstream gRPC codebase is relatively bare bones.
This package, and most of its child packages provides commonly needed middleware for gRPC:
client-side interceptors for retires, server-side interceptors for input validation and auth,
functions for chaining said interceptors, metadata convenience methods and more.
### Chaining
By default, gRPC doesn't allow one to have more than one interceptor either on the client nor on
the server side. `grpc_middleware` provides convenient chaining methods
Simple way of turning a multiple interceptors into a single interceptor. Here's an example for
server chaining:
myServer := grpc.NewServer(
grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(loggingStream, monitoringStream, authStream)),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(loggingUnary, monitoringUnary, authUnary),
)
These interceptors will be executed from left to right: logging, monitoring and auth.
Here's an example for client side chaining:
clientConn, err = grpc.Dial(
address,
grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(monitoringClientUnary, retryUnary)),
grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(monitoringClientStream, retryStream)),
)
client = pb_testproto.NewTestServiceClient(clientConn)
resp, err := client.PingEmpty(s.ctx, &myservice.Request{Msg: "hello"})
These interceptors will be executed from left to right: monitoring and then retry logic.
The retry interceptor will call every interceptor that follows it whenever when a retry happens.
### Writing Your Own
Implementing your own interceptor is pretty trivial: there are interfaces for that. But the interesting
bit exposing common data to handlers (and other middleware), similarly to HTTP Middleware design.
For example, you may want to pass the identity of the caller from the auth interceptor all the way
to the handling function.
For example, a client side interceptor example for auth looks like:
func FakeAuthUnaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
newCtx := context.WithValue(ctx, "user_id", "john@example.com")
return handler(newCtx, req)
}
Unfortunately, it's not as easy for streaming RPCs. These have the `context.Context` embedded within
the `grpc.ServerStream` object. To pass values through context, a wrapper (`WrappedServerStream`) is
needed. For example:
func FakeAuthStreamingInterceptor(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
newStream := grpc_middleware.WrapServerStream(stream)
newStream.WrappedContext = context.WithValue(ctx, "user_id", "john@example.com")
return handler(srv, stream)
}
## <a name="pkg-imports">Imported Packages</a>
- [golang.org/x/net/context](https://godoc.org/golang.org/x/net/context)
- [google.golang.org/grpc](https://godoc.org/google.golang.org/grpc)
## <a name="pkg-index">Index</a>
* [func ChainStreamClient(interceptors ...grpc.StreamClientInterceptor) grpc.StreamClientInterceptor](#ChainStreamClient)
* [func ChainStreamServer(interceptors ...grpc.StreamServerInterceptor) grpc.StreamServerInterceptor](#ChainStreamServer)
* [func ChainUnaryClient(interceptors ...grpc.UnaryClientInterceptor) grpc.UnaryClientInterceptor](#ChainUnaryClient)
* [func ChainUnaryServer(interceptors ...grpc.UnaryServerInterceptor) grpc.UnaryServerInterceptor](#ChainUnaryServer)
* [func WithStreamServerChain(interceptors ...grpc.StreamServerInterceptor) grpc.ServerOption](#WithStreamServerChain)
* [func WithUnaryServerChain(interceptors ...grpc.UnaryServerInterceptor) grpc.ServerOption](#WithUnaryServerChain)
* [type WrappedServerStream](#WrappedServerStream)
* [func WrapServerStream(stream grpc.ServerStream) \*WrappedServerStream](#WrapServerStream)
* [func (w \*WrappedServerStream) Context() context.Context](#WrappedServerStream.Context)
#### <a name="pkg-files">Package files</a>
[chain.go](./chain.go) [doc.go](./doc.go) [wrappers.go](./wrappers.go)
## <a name="ChainStreamClient">func</a> [ChainStreamClient](./chain.go#L136)
``` go
func ChainStreamClient(interceptors ...grpc.StreamClientInterceptor) grpc.StreamClientInterceptor
```
ChainStreamClient creates a single interceptor out of a chain of many interceptors.
Execution is done in left-to-right order, including passing of context.
For example ChainStreamClient(one, two, three) will execute one before two before three.
## <a name="ChainStreamServer">func</a> [ChainStreamServer](./chain.go#L58)
``` go
func ChainStreamServer(interceptors ...grpc.StreamServerInterceptor) grpc.StreamServerInterceptor
```
ChainStreamServer creates a single interceptor out of a chain of many interceptors.
Execution is done in left-to-right order, including passing of context.
For example ChainUnaryServer(one, two, three) will execute one before two before three.
If you want to pass context between interceptors, use WrapServerStream.
## <a name="ChainUnaryClient">func</a> [ChainUnaryClient](./chain.go#L97)
``` go
func ChainUnaryClient(interceptors ...grpc.UnaryClientInterceptor) grpc.UnaryClientInterceptor
```
ChainUnaryClient creates a single interceptor out of a chain of many interceptors.
Execution is done in left-to-right order, including passing of context.
For example ChainUnaryClient(one, two, three) will execute one before two before three.
## <a name="ChainUnaryServer">func</a> [ChainUnaryServer](./chain.go#L18)
``` go
func ChainUnaryServer(interceptors ...grpc.UnaryServerInterceptor) grpc.UnaryServerInterceptor
```
ChainUnaryServer creates a single interceptor out of a chain of many interceptors.
Execution is done in left-to-right order, including passing of context.
For example ChainUnaryServer(one, two, three) will execute one before two before three, and three
will see context changes of one and two.
## <a name="WithStreamServerChain">func</a> [WithStreamServerChain](./chain.go#L181)
``` go
func WithStreamServerChain(interceptors ...grpc.StreamServerInterceptor) grpc.ServerOption
```
WithStreamServerChain is a grpc.Server config option that accepts multiple stream interceptors.
Basically syntactic sugar.
## <a name="WithUnaryServerChain">func</a> [WithUnaryServerChain](./chain.go#L175)
``` go
func WithUnaryServerChain(interceptors ...grpc.UnaryServerInterceptor) grpc.ServerOption
```
Chain creates a single interceptor out of a chain of many interceptors.
WithUnaryServerChain is a grpc.Server config option that accepts multiple unary interceptors.
Basically syntactic sugar.
## <a name="WrappedServerStream">type</a> [WrappedServerStream](./wrappers.go#L12-L16)
``` go
type WrappedServerStream struct {
grpc.ServerStream
// WrappedContext is the wrapper's own Context. You can assign it.
WrappedContext context.Context
}
```
WrappedServerStream is a thin wrapper around grpc.ServerStream that allows modifying context.
### <a name="WrapServerStream">func</a> [WrapServerStream](./wrappers.go#L24)
``` go
func WrapServerStream(stream grpc.ServerStream) *WrappedServerStream
```
WrapServerStream returns a ServerStream that has the ability to overwrite context.
### <a name="WrappedServerStream.Context">func</a> (\*WrappedServerStream) [Context](./wrappers.go#L19)
``` go
func (w *WrappedServerStream) Context() context.Context
```
Context returns the wrapper's WrappedContext, overwriting the nested grpc.ServerStream.Context()
- - -
Generated by [godoc2ghmd](https://github.com/GandalfUK/godoc2ghmd)
\ No newline at end of file
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "cloud.google.com/go"
packages = ["compute/metadata"]
revision = "2d3a6656c17a60b0815b7e06ab0be04eacb6e613"
version = "v0.16.0"
[[projects]]
name = "github.com/davecgh/go-spew"
packages = ["spew"]
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
name = "github.com/gogo/protobuf"
packages = ["gogoproto","proto","protoc-gen-gogo/descriptor"]
revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02"
version = "v0.5"
[[projects]]
branch = "master"
name = "github.com/golang/protobuf"
packages = ["jsonpb","proto","ptypes","ptypes/any","ptypes/duration","ptypes/struct","ptypes/timestamp"]
revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845"
[[projects]]
name = "github.com/opentracing/opentracing-go"
packages = [".","ext","log","mocktracer"]
revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
version = "v1.0.2"
[[projects]]
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]]
name = "github.com/sirupsen/logrus"
packages = ["."]
revision = "f006c2ac4710855cf0f916dd6b77acf6b048dc6e"
version = "v1.0.3"
[[projects]]
name = "github.com/stretchr/testify"
packages = ["assert","require","suite"]
revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0"
version = "v1.1.4"
[[projects]]
name = "go.uber.org/atomic"
packages = ["."]
revision = "8474b86a5a6f79c443ce4b2992817ff32cf208b8"
version = "v1.3.1"
[[projects]]
name = "go.uber.org/multierr"
packages = ["."]
revision = "3c4937480c32f4c13a875a1829af76c98ca3d40a"
version = "v1.1.0"
[[projects]]
name = "go.uber.org/zap"
packages = [".","buffer","internal/bufferpool","internal/color","internal/exit","zapcore"]
revision = "35aad584952c3e7020db7b839f6b102de6271f89"
version = "v1.7.1"
[[projects]]
branch = "master"
name = "golang.org/x/crypto"
packages = ["ssh/terminal"]
revision = "94eea52f7b742c7cbe0b03b22f0c4c8631ece122"
[[projects]]
branch = "master"
name = "golang.org/x/net"
packages = ["context","context/ctxhttp","http2","http2/hpack","idna","internal/timeseries","lex/httplex","trace"]
revision = "a8b9294777976932365dabb6640cf1468d95c70f"
[[projects]]
branch = "master"
name = "golang.org/x/oauth2"
packages = [".","google","internal","jws","jwt"]
revision = "f95fa95eaa936d9d87489b15d1d18b97c1ba9c28"
[[projects]]
branch = "master"
name = "golang.org/x/sys"
packages = ["unix","windows"]
revision = "13fcbd661c8ececa8807a29b48407d674b1d8ed8"
[[projects]]
branch = "master"
name = "golang.org/x/text"
packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
revision = "75cc3cad82b5f47d3fb229ddda8c5167da14f294"
[[projects]]
name = "google.golang.org/appengine"
packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/urlfetch","urlfetch"]
revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "google.golang.org/genproto"
packages = ["googleapis/rpc/status"]
revision = "7f0da29060c682909f650ad8ed4e515bd74fa12a"
[[projects]]
name = "google.golang.org/grpc"
packages = [".","balancer","balancer/roundrobin","codes","connectivity","credentials","credentials/oauth","encoding","grpclb/grpc_lb_v1/messages","grpclog","internal","keepalive","metadata","naming","peer","resolver","resolver/dns","resolver/passthrough","stats","status","tap","transport"]
revision = "5a9f7b402fe85096d2e1d0383435ee1876e863d0"
version = "v1.8.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "b24c6670412eb0bc44ed1db77fecc52333f8725f3e3272bdc568f5683a63031f"
solver-name = "gps-cdcl"
solver-version = 1
[[constraint]]
name = "github.com/gogo/protobuf"
version = "0.5.0"
[[constraint]]
branch = "master"
name = "github.com/golang/protobuf"
[[constraint]]
name = "github.com/opentracing/opentracing-go"
version = "1.0.2"
[[constraint]]
name = "github.com/sirupsen/logrus"
version = "1.0.3"
[[constraint]]
name = "github.com/stretchr/testify"
version = "1.1.4"
[[constraint]]
name = "go.uber.org/zap"
version = "1.7.1"
[[constraint]]
branch = "master"
name = "golang.org/x/net"
[[constraint]]
branch = "master"
name = "golang.org/x/oauth2"
[[constraint]]
name = "google.golang.org/grpc"
version = "1.8.0"
# Go gRPC Middleware
[![Travis Build](https://travis-ci.org/grpc-ecosystem/go-grpc-middleware.svg?branch=master)](https://travis-ci.org/grpc-ecosystem/go-grpc-middleware)
[![Go Report Card](https://goreportcard.com/badge/github.com/grpc-ecosystem/go-grpc-middleware)](https://goreportcard.com/report/github.com/grpc-ecosystem/go-grpc-middleware)
[![GoDoc](http://img.shields.io/badge/GoDoc-Reference-blue.svg)](https://godoc.org/github.com/grpc-ecosystem/go-grpc-middleware)
[![SourceGraph](https://sourcegraph.com/github.com/grpc-ecosystem/go-grpc-middleware/-/badge.svg)](https://sourcegraph.com/github.com/grpc-ecosystem/go-grpc-middleware/?badge)
[![codecov](https://codecov.io/gh/grpc-ecosystem/go-grpc-middleware/branch/master/graph/badge.svg)](https://codecov.io/gh/grpc-ecosystem/go-grpc-middleware)
[![Apache 2.0 License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
[![quality: production](https://img.shields.io/badge/quality-production-orange.svg)](#status)
[![Slack](slack.png)](https://join.slack.com/t/improbable-eng/shared_invite/enQtMzQ1ODcyMzQ5MjM4LWY5ZWZmNGM2ODc5MmViNmQ3ZTA3ZTY3NzQwOTBlMTkzZmIxZTIxODk0OWU3YjZhNWVlNDU3MDlkZGViZjhkMjc)
[gRPC Go](https://github.com/grpc/grpc-go) Middleware: interceptors, helpers, utilities.
**Important** The repo recently moved to `github.com/grpc-ecosystem/go-grpc-middleware`, please update your import paths.
## Middleware
[gRPC Go](https://github.com/grpc/grpc-go) recently acquired support for
Interceptors, i.e. [middleware](https://medium.com/@matryer/writing-middleware-in-golang-and-how-go-makes-it-so-much-fun-4375c1246e81#.gv7tdlghs)
that is executed either on the gRPC Server before the request is passed onto the user's application logic, or on the gRPC client either around the user call. It is a perfect way to implement
common patterns: auth, logging, message, validation, retries or monitoring.
These are generic building blocks that make it easy to build multiple microservices easily.
The purpose of this repository is to act as a go-to point for such reusable functionality. It contains
some of them itself, but also will link to useful external repos.
`grpc_middleware` itself provides support for chaining interceptors. See [Documentation](DOC.md), but here's an example:
```go
import "github.com/grpc-ecosystem/go-grpc-middleware"
myServer := grpc.NewServer(
grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
grpc_ctxtags.StreamServerInterceptor(),
grpc_opentracing.StreamServerInterceptor(),
grpc_prometheus.StreamServerInterceptor,
grpc_zap.StreamServerInterceptor(zapLogger),
grpc_auth.StreamServerInterceptor(myAuthFunction),
grpc_recovery.StreamServerInterceptor(),
)),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
grpc_ctxtags.UnaryServerInterceptor(),
grpc_opentracing.UnaryServerInterceptor(),
grpc_prometheus.UnaryServerInterceptor,
grpc_zap.UnaryServerInterceptor(zapLogger),
grpc_auth.UnaryServerInterceptor(myAuthFunction),
grpc_recovery.UnaryServerInterceptor(),
)),
)
```
## Interceptors
*Please send a PR to add new interceptors or middleware to this list*
#### Auth
* [`grpc_auth`](auth) - a customizable (via `AuthFunc`) piece of auth middleware
#### Logging
* [`grpc_ctxtags`](tags/) - a library that adds a `Tag` map to context, with data populated from request body
* [`grpc_zap`](logging/zap/) - integration of [zap](https://github.com/uber-go/zap) logging library into gRPC handlers.
* [`grpc_logrus`](logging/logrus/) - integration of [logrus](https://github.com/sirupsen/logrus) logging library into gRPC handlers.
#### Monitoring
* [`grpc_prometheus`⚡](https://github.com/grpc-ecosystem/go-grpc-prometheus) - Prometheus client-side and server-side monitoring middleware
* [`otgrpc`⚡](https://github.com/grpc-ecosystem/grpc-opentracing/tree/master/go/otgrpc) - [OpenTracing](http://opentracing.io/) client-side and server-side interceptors
* [`grpc_opentracing`](tracing/opentracing) - [OpenTracing](http://opentracing.io/) client-side and server-side interceptors with support for streaming and handler-returned tags
#### Client
* [`grpc_retry`](retry/) - a generic gRPC response code retry mechanism, client-side middleware
#### Server
* [`grpc_validator`](validator/) - codegen inbound message validation from `.proto` options
* [`grpc_recovery`](recovery/) - turn panics into gRPC errors
## Status
This code has been running in *production* since May 2016 as the basis of the gRPC micro services stack at [Improbable](https://improbable.io).
Additional tooling will be added, and contributions are welcome.
## License
`go-grpc-middleware` is released under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details.
// Copyright 2016 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.
// gRPC Server Interceptor chaining middleware.
package grpc_middleware
import (
"golang.org/x/net/context"
"google.golang.org/grpc"
)
// ChainUnaryServer creates a single interceptor out of a chain of many interceptors.
//
// Execution is done in left-to-right order, including passing of context.
// For example ChainUnaryServer(one, two, three) will execute one before two before three, and three
// will see context changes of one and two.
func ChainUnaryServer(interceptors ...grpc.UnaryServerInterceptor) grpc.UnaryServerInterceptor {
n := len(interceptors)
if n > 1 {
lastI := n - 1
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
var (
chainHandler grpc.UnaryHandler
curI int
)
chainHandler = func(currentCtx context.Context, currentReq interface{}) (interface{}, error) {
if curI == lastI {
return handler(currentCtx, currentReq)
}
curI++
resp, err := interceptors[curI](currentCtx, currentReq, info, chainHandler)
curI--
return resp, err
}
return interceptors[0](ctx, req, info, chainHandler)
}
}
if n == 1 {
return interceptors[0]
}
// n == 0; Dummy interceptor maintained for backward compatibility to avoid returning nil.
return func(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
return handler(ctx, req)
}
}
// ChainStreamServer creates a single interceptor out of a chain of many interceptors.
//
// Execution is done in left-to-right order, including passing of context.
// For example ChainUnaryServer(one, two, three) will execute one before two before three.
// If you want to pass context between interceptors, use WrapServerStream.
func ChainStreamServer(interceptors ...grpc.StreamServerInterceptor) grpc.StreamServerInterceptor {
n := len(interceptors)
if n > 1 {
lastI := n - 1
return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
var (
chainHandler grpc.StreamHandler
curI int
)
chainHandler = func(currentSrv interface{}, currentStream grpc.ServerStream) error {
if curI == lastI {
return handler(currentSrv, currentStream)
}
curI++
err := interceptors[curI](currentSrv, currentStream, info, chainHandler)
curI--
return err
}
return interceptors[0](srv, stream, info, chainHandler)
}
}
if n == 1 {
return interceptors[0]
}
// n == 0; Dummy interceptor maintained for backward compatibility to avoid returning nil.
return func(srv interface{}, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
return handler(srv, stream)
}
}
// ChainUnaryClient creates a single interceptor out of a chain of many interceptors.
//
// Execution is done in left-to-right order, including passing of context.
// For example ChainUnaryClient(one, two, three) will execute one before two before three.
func ChainUnaryClient(interceptors ...grpc.UnaryClientInterceptor) grpc.UnaryClientInterceptor {
n := len(interceptors)
if n > 1 {
lastI := n - 1
return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
var (
chainHandler grpc.UnaryInvoker
curI int
)
chainHandler = func(currentCtx context.Context, currentMethod string, currentReq, currentRepl interface{}, currentConn *grpc.ClientConn, currentOpts ...grpc.CallOption) error {
if curI == lastI {
return invoker(currentCtx, currentMethod, currentReq, currentRepl, currentConn, currentOpts...)
}
curI++
err := interceptors[curI](currentCtx, currentMethod, currentReq, currentRepl, currentConn, chainHandler, currentOpts...)
curI--
return err
}
return interceptors[0](ctx, method, req, reply, cc, chainHandler, opts...)
}
}
if n == 1 {
return interceptors[0]
}
// n == 0; Dummy interceptor maintained for backward compatibility to avoid returning nil.
return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
return invoker(ctx, method, req, reply, cc, opts...)
}
}
// ChainStreamClient creates a single interceptor out of a chain of many interceptors.
//
// Execution is done in left-to-right order, including passing of context.
// For example ChainStreamClient(one, two, three) will execute one before two before three.
func ChainStreamClient(interceptors ...grpc.StreamClientInterceptor) grpc.StreamClientInterceptor {
n := len(interceptors)
if n > 1 {
lastI := n - 1
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
var (
chainHandler grpc.Streamer
curI int
)
chainHandler = func(currentCtx context.Context, currentDesc *grpc.StreamDesc, currentConn *grpc.ClientConn, currentMethod string, currentOpts ...grpc.CallOption) (grpc.ClientStream, error) {
if curI == lastI {
return streamer(currentCtx, currentDesc, currentConn, currentMethod, currentOpts...)
}
curI++
stream, err := interceptors[curI](currentCtx, currentDesc, currentConn, currentMethod, chainHandler, currentOpts...)
curI--
return stream, err
}
return interceptors[0](ctx, desc, cc, method, chainHandler, opts...)
}
}
if n == 1 {
return interceptors[0]
}
// n == 0; Dummy interceptor maintained for backward compatibility to avoid returning nil.
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
return streamer(ctx, desc, cc, method, opts...)
}
}
// Chain creates a single interceptor out of a chain of many interceptors.
//
// WithUnaryServerChain is a grpc.Server config option that accepts multiple unary interceptors.
// Basically syntactic sugar.
func WithUnaryServerChain(interceptors ...grpc.UnaryServerInterceptor) grpc.ServerOption {
return grpc.UnaryInterceptor(ChainUnaryServer(interceptors...))
}
// WithStreamServerChain is a grpc.Server config option that accepts multiple stream interceptors.
// Basically syntactic sugar.
func WithStreamServerChain(interceptors ...grpc.StreamServerInterceptor) grpc.ServerOption {
return grpc.StreamInterceptor(ChainStreamServer(interceptors...))
}
// Copyright 2016 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.
/*
`grpc_middleware` is a collection of gRPC middleware packages: interceptors, helpers and tools.
Middleware
gRPC is a fantastic RPC middleware, which sees a lot of adoption in the Golang world. However, the
upstream gRPC codebase is relatively bare bones.
This package, and most of its child packages provides commonly needed middleware for gRPC:
client-side interceptors for retires, server-side interceptors for input validation and auth,
functions for chaining said interceptors, metadata convenience methods and more.
Chaining
By default, gRPC doesn't allow one to have more than one interceptor either on the client nor on
the server side. `grpc_middleware` provides convenient chaining methods
Simple way of turning a multiple interceptors into a single interceptor. Here's an example for
server chaining:
myServer := grpc.NewServer(
grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(loggingStream, monitoringStream, authStream)),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(loggingUnary, monitoringUnary, authUnary),
)
These interceptors will be executed from left to right: logging, monitoring and auth.
Here's an example for client side chaining:
clientConn, err = grpc.Dial(
address,
grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(monitoringClientUnary, retryUnary)),
grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(monitoringClientStream, retryStream)),
)
client = pb_testproto.NewTestServiceClient(clientConn)
resp, err := client.PingEmpty(s.ctx, &myservice.Request{Msg: "hello"})
These interceptors will be executed from left to right: monitoring and then retry logic.
The retry interceptor will call every interceptor that follows it whenever when a retry happens.
Writing Your Own
Implementing your own interceptor is pretty trivial: there are interfaces for that. But the interesting
bit exposing common data to handlers (and other middleware), similarly to HTTP Middleware design.
For example, you may want to pass the identity of the caller from the auth interceptor all the way
to the handling function.
For example, a client side interceptor example for auth looks like:
func FakeAuthUnaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
newCtx := context.WithValue(ctx, "user_id", "john@example.com")
return handler(newCtx, req)
}
Unfortunately, it's not as easy for streaming RPCs. These have the `context.Context` embedded within
the `grpc.ServerStream` object. To pass values through context, a wrapper (`WrappedServerStream`) is
needed. For example:
func FakeAuthStreamingInterceptor(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
newStream := grpc_middleware.WrapServerStream(stream)
newStream.WrappedContext = context.WithValue(ctx, "user_id", "john@example.com")
return handler(srv, stream)
}
*/
package grpc_middleware
SHELL="/bin/bash"
GOFILES_NOVENDOR = $(shell go list ./... | grep -v /vendor/)
all: vet fmt docs test
docs:
./scripts/docs.sh generate
checkdocs:
./scripts/docs.sh check
fmt:
go fmt $(GOFILES_NOVENDOR)
vet:
go vet $(GOFILES_NOVENDOR)
test: vet
./scripts/test_all.sh
.PHONY: all docs validate test
// Copyright 2016 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.
package grpc_middleware
import (
"golang.org/x/net/context"
"google.golang.org/grpc"
)
// WrappedServerStream is a thin wrapper around grpc.ServerStream that allows modifying context.
type WrappedServerStream struct {
grpc.ServerStream
// WrappedContext is the wrapper's own Context. You can assign it.
WrappedContext context.Context
}
// Context returns the wrapper's WrappedContext, overwriting the nested grpc.ServerStream.Context()
func (w *WrappedServerStream) Context() context.Context {
return w.WrappedContext
}
// WrapServerStream returns a ServerStream that has the ability to overwrite context.
func WrapServerStream(stream grpc.ServerStream) *WrappedServerStream {
if existing, ok := stream.(*WrappedServerStream); ok {
return existing
}
return &WrappedServerStream{ServerStream: stream, WrappedContext: stream.Context()}
}
coverage:
range: 80..100
round: down
precision: 2
status:
project: # measuring the overall project coverage
default: # context, you can create multiple ones with custom titles
enabled: yes # must be yes|true to enable this status
target: 100 # specify the target coverage for each commit status
# option: "auto" (must increase from parent commit or pull request base)
# option: "X%" a static target percentage to hit
if_not_found: success # if parent is not found report status as success, error, or failure
if_ci_failed: error # if ci fails report status as success, error, or failure
.DS_Store
/vendor
/cover
cover.out
lint.log
# Binaries
*.test
# Profiling output
*.prof
sudo: false
language: go
go_import_path: go.uber.org/atomic
go:
- 1.7.x
- 1.8.x
- 1.9.x
- 1.10.x
- 1.x # latest release
cache:
directories:
- vendor
install:
- make install_ci
script:
- make test_ci
- scripts/test-ubergo.sh
- make lint
after_success:
- bash <(curl -s https://codecov.io/bash)
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"atomic.go",
"error.go",
"string.go",
],
importmap = "k8s.io/kubernetes/vendor/go.uber.org/atomic",
importpath = "go.uber.org/atomic",
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
Copyright (c) 2016 Uber Technologies, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
PACKAGES := $(shell glide nv)
# Many Go tools take file globs or directories as arguments instead of packages.
PACKAGE_FILES ?= *.go
# The linting tools evolve with each Go version, so run them only on the latest
# stable release.
GO_VERSION := $(shell go version | cut -d " " -f 3)
GO_MINOR_VERSION := $(word 2,$(subst ., ,$(GO_VERSION)))
LINTABLE_MINOR_VERSIONS := 7 8
ifneq ($(filter $(LINTABLE_MINOR_VERSIONS),$(GO_MINOR_VERSION)),)
SHOULD_LINT := true
endif
export GO15VENDOREXPERIMENT=1
.PHONY: build
build:
go build -i $(PACKAGES)
.PHONY: install
install:
glide --version || go get github.com/Masterminds/glide
glide install
.PHONY: test
test:
go test -cover -race $(PACKAGES)
.PHONY: install_ci
install_ci: install
go get github.com/wadey/gocovmerge
go get github.com/mattn/goveralls
go get golang.org/x/tools/cmd/cover
ifdef SHOULD_LINT
go get github.com/golang/lint/golint
endif
.PHONY: lint
lint:
ifdef SHOULD_LINT
@rm -rf lint.log
@echo "Checking formatting..."
@gofmt -d -s $(PACKAGE_FILES) 2>&1 | tee lint.log
@echo "Checking vet..."
@$(foreach dir,$(PACKAGE_FILES),go tool vet $(dir) 2>&1 | tee -a lint.log;)
@echo "Checking lint..."
@$(foreach dir,$(PKGS),golint $(dir) 2>&1 | tee -a lint.log;)
@echo "Checking for unresolved FIXMEs..."
@git grep -i fixme | grep -v -e vendor -e Makefile | tee -a lint.log
@[ ! -s lint.log ]
else
@echo "Skipping linters on" $(GO_VERSION)
endif
.PHONY: test_ci
test_ci: install_ci build
./scripts/cover.sh $(shell go list $(PACKAGES))
# atomic [![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Coverage Status][cov-img]][cov] [![Go Report Card][reportcard-img]][reportcard]
Simple wrappers for primitive types to enforce atomic access.
## Installation
`go get -u go.uber.org/atomic`
## Usage
The standard library's `sync/atomic` is powerful, but it's easy to forget which
variables must be accessed atomically. `go.uber.org/atomic` preserves all the
functionality of the standard library, but wraps the primitive types to
provide a safer, more convenient API.
```go
var atom atomic.Uint32
atom.Store(42)
atom.Sub(2)
atom.CAS(40, 11)
```
See the [documentation][doc] for a complete API specification.
## Development Status
Stable.
___
Released under the [MIT License](LICENSE.txt).
[doc-img]: https://godoc.org/github.com/uber-go/atomic?status.svg
[doc]: https://godoc.org/go.uber.org/atomic
[ci-img]: https://travis-ci.org/uber-go/atomic.svg?branch=master
[ci]: https://travis-ci.org/uber-go/atomic
[cov-img]: https://codecov.io/gh/uber-go/atomic/branch/master/graph/badge.svg
[cov]: https://codecov.io/gh/uber-go/atomic
[reportcard-img]: https://goreportcard.com/badge/go.uber.org/atomic
[reportcard]: https://goreportcard.com/report/go.uber.org/atomic
// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
// Package atomic provides simple wrappers around numerics to enforce atomic
// access.
package atomic
import (
"math"
"sync/atomic"
"time"
)
// Int32 is an atomic wrapper around an int32.
type Int32 struct{ v int32 }
// NewInt32 creates an Int32.
func NewInt32(i int32) *Int32 {
return &Int32{i}
}
// Load atomically loads the wrapped value.
func (i *Int32) Load() int32 {
return atomic.LoadInt32(&i.v)
}
// Add atomically adds to the wrapped int32 and returns the new value.
func (i *Int32) Add(n int32) int32 {
return atomic.AddInt32(&i.v, n)
}
// Sub atomically subtracts from the wrapped int32 and returns the new value.
func (i *Int32) Sub(n int32) int32 {
return atomic.AddInt32(&i.v, -n)
}
// Inc atomically increments the wrapped int32 and returns the new value.
func (i *Int32) Inc() int32 {
return i.Add(1)
}
// Dec atomically decrements the wrapped int32 and returns the new value.
func (i *Int32) Dec() int32 {
return i.Sub(1)
}
// CAS is an atomic compare-and-swap.
func (i *Int32) CAS(old, new int32) bool {
return atomic.CompareAndSwapInt32(&i.v, old, new)
}
// Store atomically stores the passed value.
func (i *Int32) Store(n int32) {
atomic.StoreInt32(&i.v, n)
}
// Swap atomically swaps the wrapped int32 and returns the old value.
func (i *Int32) Swap(n int32) int32 {
return atomic.SwapInt32(&i.v, n)
}
// Int64 is an atomic wrapper around an int64.
type Int64 struct{ v int64 }
// NewInt64 creates an Int64.
func NewInt64(i int64) *Int64 {
return &Int64{i}
}
// Load atomically loads the wrapped value.
func (i *Int64) Load() int64 {
return atomic.LoadInt64(&i.v)
}
// Add atomically adds to the wrapped int64 and returns the new value.
func (i *Int64) Add(n int64) int64 {
return atomic.AddInt64(&i.v, n)
}
// Sub atomically subtracts from the wrapped int64 and returns the new value.
func (i *Int64) Sub(n int64) int64 {
return atomic.AddInt64(&i.v, -n)
}
// Inc atomically increments the wrapped int64 and returns the new value.
func (i *Int64) Inc() int64 {
return i.Add(1)
}
// Dec atomically decrements the wrapped int64 and returns the new value.
func (i *Int64) Dec() int64 {
return i.Sub(1)
}
// CAS is an atomic compare-and-swap.
func (i *Int64) CAS(old, new int64) bool {
return atomic.CompareAndSwapInt64(&i.v, old, new)
}
// Store atomically stores the passed value.
func (i *Int64) Store(n int64) {
atomic.StoreInt64(&i.v, n)
}
// Swap atomically swaps the wrapped int64 and returns the old value.
func (i *Int64) Swap(n int64) int64 {
return atomic.SwapInt64(&i.v, n)
}
// Uint32 is an atomic wrapper around an uint32.
type Uint32 struct{ v uint32 }
// NewUint32 creates a Uint32.
func NewUint32(i uint32) *Uint32 {
return &Uint32{i}
}
// Load atomically loads the wrapped value.
func (i *Uint32) Load() uint32 {
return atomic.LoadUint32(&i.v)
}
// Add atomically adds to the wrapped uint32 and returns the new value.
func (i *Uint32) Add(n uint32) uint32 {
return atomic.AddUint32(&i.v, n)
}
// Sub atomically subtracts from the wrapped uint32 and returns the new value.
func (i *Uint32) Sub(n uint32) uint32 {
return atomic.AddUint32(&i.v, ^(n - 1))
}
// Inc atomically increments the wrapped uint32 and returns the new value.
func (i *Uint32) Inc() uint32 {
return i.Add(1)
}
// Dec atomically decrements the wrapped int32 and returns the new value.
func (i *Uint32) Dec() uint32 {
return i.Sub(1)
}
// CAS is an atomic compare-and-swap.
func (i *Uint32) CAS(old, new uint32) bool {
return atomic.CompareAndSwapUint32(&i.v, old, new)
}
// Store atomically stores the passed value.
func (i *Uint32) Store(n uint32) {
atomic.StoreUint32(&i.v, n)
}
// Swap atomically swaps the wrapped uint32 and returns the old value.
func (i *Uint32) Swap(n uint32) uint32 {
return atomic.SwapUint32(&i.v, n)
}
// Uint64 is an atomic wrapper around a uint64.
type Uint64 struct{ v uint64 }
// NewUint64 creates a Uint64.
func NewUint64(i uint64) *Uint64 {
return &Uint64{i}
}
// Load atomically loads the wrapped value.
func (i *Uint64) Load() uint64 {
return atomic.LoadUint64(&i.v)
}
// Add atomically adds to the wrapped uint64 and returns the new value.
func (i *Uint64) Add(n uint64) uint64 {
return atomic.AddUint64(&i.v, n)
}
// Sub atomically subtracts from the wrapped uint64 and returns the new value.
func (i *Uint64) Sub(n uint64) uint64 {
return atomic.AddUint64(&i.v, ^(n - 1))
}
// Inc atomically increments the wrapped uint64 and returns the new value.
func (i *Uint64) Inc() uint64 {
return i.Add(1)
}
// Dec atomically decrements the wrapped uint64 and returns the new value.
func (i *Uint64) Dec() uint64 {
return i.Sub(1)
}
// CAS is an atomic compare-and-swap.
func (i *Uint64) CAS(old, new uint64) bool {
return atomic.CompareAndSwapUint64(&i.v, old, new)
}
// Store atomically stores the passed value.
func (i *Uint64) Store(n uint64) {
atomic.StoreUint64(&i.v, n)
}
// Swap atomically swaps the wrapped uint64 and returns the old value.
func (i *Uint64) Swap(n uint64) uint64 {
return atomic.SwapUint64(&i.v, n)
}
// Bool is an atomic Boolean.
type Bool struct{ v uint32 }
// NewBool creates a Bool.
func NewBool(initial bool) *Bool {
return &Bool{boolToInt(initial)}
}
// Load atomically loads the Boolean.
func (b *Bool) Load() bool {
return truthy(atomic.LoadUint32(&b.v))
}
// CAS is an atomic compare-and-swap.
func (b *Bool) CAS(old, new bool) bool {
return atomic.CompareAndSwapUint32(&b.v, boolToInt(old), boolToInt(new))
}
// Store atomically stores the passed value.
func (b *Bool) Store(new bool) {
atomic.StoreUint32(&b.v, boolToInt(new))
}
// Swap sets the given value and returns the previous value.
func (b *Bool) Swap(new bool) bool {
return truthy(atomic.SwapUint32(&b.v, boolToInt(new)))
}
// Toggle atomically negates the Boolean and returns the previous value.
func (b *Bool) Toggle() bool {
return truthy(atomic.AddUint32(&b.v, 1) - 1)
}
func truthy(n uint32) bool {
return n&1 == 1
}
func boolToInt(b bool) uint32 {
if b {
return 1
}
return 0
}
// Float64 is an atomic wrapper around float64.
type Float64 struct {
v uint64
}
// NewFloat64 creates a Float64.
func NewFloat64(f float64) *Float64 {
return &Float64{math.Float64bits(f)}
}
// Load atomically loads the wrapped value.
func (f *Float64) Load() float64 {
return math.Float64frombits(atomic.LoadUint64(&f.v))
}
// Store atomically stores the passed value.
func (f *Float64) Store(s float64) {
atomic.StoreUint64(&f.v, math.Float64bits(s))
}
// Add atomically adds to the wrapped float64 and returns the new value.
func (f *Float64) Add(s float64) float64 {
for {
old := f.Load()
new := old + s
if f.CAS(old, new) {
return new
}
}
}
// Sub atomically subtracts from the wrapped float64 and returns the new value.
func (f *Float64) Sub(s float64) float64 {
return f.Add(-s)
}
// CAS is an atomic compare-and-swap.
func (f *Float64) CAS(old, new float64) bool {
return atomic.CompareAndSwapUint64(&f.v, math.Float64bits(old), math.Float64bits(new))
}
// Duration is an atomic wrapper around time.Duration
// https://godoc.org/time#Duration
type Duration struct {
v Int64
}
// NewDuration creates a Duration.
func NewDuration(d time.Duration) *Duration {
return &Duration{v: *NewInt64(int64(d))}
}
// Load atomically loads the wrapped value.
func (d *Duration) Load() time.Duration {
return time.Duration(d.v.Load())
}
// Store atomically stores the passed value.
func (d *Duration) Store(n time.Duration) {
d.v.Store(int64(n))
}
// Add atomically adds to the wrapped time.Duration and returns the new value.
func (d *Duration) Add(n time.Duration) time.Duration {
return time.Duration(d.v.Add(int64(n)))
}
// Sub atomically subtracts from the wrapped time.Duration and returns the new value.
func (d *Duration) Sub(n time.Duration) time.Duration {
return time.Duration(d.v.Sub(int64(n)))
}
// Swap atomically swaps the wrapped time.Duration and returns the old value.
func (d *Duration) Swap(n time.Duration) time.Duration {
return time.Duration(d.v.Swap(int64(n)))
}
// CAS is an atomic compare-and-swap.
func (d *Duration) CAS(old, new time.Duration) bool {
return d.v.CAS(int64(old), int64(new))
}
// Value shadows the type of the same name from sync/atomic
// https://godoc.org/sync/atomic#Value
type Value struct{ atomic.Value }
// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package atomic
// Error is an atomic type-safe wrapper around Value for errors
type Error struct{ v Value }
// errorHolder is non-nil holder for error object.
// atomic.Value panics on saving nil object, so err object needs to be
// wrapped with valid object first.
type errorHolder struct{ err error }
// NewError creates new atomic error object
func NewError(err error) *Error {
e := &Error{}
if err != nil {
e.Store(err)
}
return e
}
// Load atomically loads the wrapped error
func (e *Error) Load() error {
v := e.v.Load()
if v == nil {
return nil
}
eh := v.(errorHolder)
return eh.err
}
// Store atomically stores error.
// NOTE: a holder object is allocated on each Store call.
func (e *Error) Store(err error) {
e.v.Store(errorHolder{err: err})
}
hash: f14d51408e3e0e4f73b34e4039484c78059cd7fc5f4996fdd73db20dc8d24f53
updated: 2016-10-27T00:10:51.16960137-07:00
imports: []
testImports:
- name: github.com/davecgh/go-spew
version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d
subpackages:
- spew
- name: github.com/pmezard/go-difflib
version: d8ed2627bdf02c080bf22230dbb337003b7aba2d
subpackages:
- difflib
- name: github.com/stretchr/testify
version: d77da356e56a7428ad25149ca77381849a6a5232
subpackages:
- assert
- require
package: go.uber.org/atomic
testImport:
- package: github.com/stretchr/testify
subpackages:
- assert
- require
// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package atomic
// String is an atomic type-safe wrapper around Value for strings.
type String struct{ v Value }
// NewString creates a String.
func NewString(str string) *String {
s := &String{}
if str != "" {
s.Store(str)
}
return s
}
// Load atomically loads the wrapped string.
func (s *String) Load() string {
v := s.v.Load()
if v == nil {
return ""
}
return v.(string)
}
// Store atomically stores the passed string.
// Note: Converting the string to an interface{} to store in the Value
// requires an allocation.
func (s *String) Store(str string) {
s.v.Store(str)
}
coverage:
range: 80..100
round: down
precision: 2
status:
project: # measuring the overall project coverage
default: # context, you can create multiple ones with custom titles
enabled: yes # must be yes|true to enable this status
target: 100 # specify the target coverage for each commit status
# option: "auto" (must increase from parent commit or pull request base)
# option: "X%" a static target percentage to hit
if_not_found: success # if parent is not found report status as success, error, or failure
if_ci_failed: error # if ci fails report status as success, error, or failure
sudo: false
language: go
go_import_path: go.uber.org/multierr
env:
global:
- GO15VENDOREXPERIMENT=1
go:
- 1.7
- 1.8
- 1.9
cache:
directories:
- vendor
before_install:
- go version
install:
- |
set -e
make install_ci
script:
- |
set -e
make lint
make test_ci
after_success:
- bash <(curl -s https://codecov.io/bash)
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["error.go"],
importmap = "k8s.io/kubernetes/vendor/go.uber.org/multierr",
importpath = "go.uber.org/multierr",
visibility = ["//visibility:public"],
deps = ["//vendor/go.uber.org/atomic:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
Releases
========
v1.1.0 (2017-06-30)
===================
- Added an `Errors(error) []error` function to extract the underlying list of
errors for a multierr error.
v1.0.0 (2017-05-31)
===================
No changes since v0.2.0. This release is committing to making no breaking
changes to the current API in the 1.X series.
v0.2.0 (2017-04-11)
===================
- Repeatedly appending to the same error is now faster due to fewer
allocations.
v0.1.0 (2017-31-03)
===================
- Initial release
Copyright (c) 2017 Uber Technologies, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
export GO15VENDOREXPERIMENT=1
PACKAGES := $(shell glide nv)
GO_FILES := $(shell \
find . '(' -path '*/.*' -o -path './vendor' ')' -prune \
-o -name '*.go' -print | cut -b3-)
.PHONY: install
install:
glide --version || go get github.com/Masterminds/glide
glide install
.PHONY: build
build:
go build -i $(PACKAGES)
.PHONY: test
test:
go test -cover -race $(PACKAGES)
.PHONY: gofmt
gofmt:
$(eval FMT_LOG := $(shell mktemp -t gofmt.XXXXX))
@gofmt -e -s -l $(GO_FILES) > $(FMT_LOG) || true
@[ ! -s "$(FMT_LOG)" ] || (echo "gofmt failed:" | cat - $(FMT_LOG) && false)
.PHONY: govet
govet:
$(eval VET_LOG := $(shell mktemp -t govet.XXXXX))
@go vet $(PACKAGES) 2>&1 \
| grep -v '^exit status' > $(VET_LOG) || true
@[ ! -s "$(VET_LOG)" ] || (echo "govet failed:" | cat - $(VET_LOG) && false)
.PHONY: golint
golint:
@go get github.com/golang/lint/golint
$(eval LINT_LOG := $(shell mktemp -t golint.XXXXX))
@cat /dev/null > $(LINT_LOG)
@$(foreach pkg, $(PACKAGES), golint $(pkg) >> $(LINT_LOG) || true;)
@[ ! -s "$(LINT_LOG)" ] || (echo "golint failed:" | cat - $(LINT_LOG) && false)
.PHONY: staticcheck
staticcheck:
@go get honnef.co/go/tools/cmd/staticcheck
$(eval STATICCHECK_LOG := $(shell mktemp -t staticcheck.XXXXX))
@staticcheck $(PACKAGES) 2>&1 > $(STATICCHECK_LOG) || true
@[ ! -s "$(STATICCHECK_LOG)" ] || (echo "staticcheck failed:" | cat - $(STATICCHECK_LOG) && false)
.PHONY: lint
lint: gofmt govet golint staticcheck
.PHONY: cover
cover:
./scripts/cover.sh $(shell go list $(PACKAGES))
go tool cover -html=cover.out -o cover.html
update-license:
@go get go.uber.org/tools/update-license
@update-license \
$(shell go list -json $(PACKAGES) | \
jq -r '.Dir + "/" + (.GoFiles | .[])')
##############################################################################
.PHONY: install_ci
install_ci: install
go get github.com/wadey/gocovmerge
go get github.com/mattn/goveralls
go get golang.org/x/tools/cmd/cover
.PHONY: test_ci
test_ci: install_ci
./scripts/cover.sh $(shell go list $(PACKAGES))
# multierr [![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Coverage Status][cov-img]][cov]
`multierr` allows combining one or more Go `error`s together.
## Installation
go get -u go.uber.org/multierr
## Status
Stable: No breaking changes will be made before 2.0.
-------------------------------------------------------------------------------
Released under the [MIT License].
[MIT License]: LICENSE.txt
[doc-img]: https://godoc.org/go.uber.org/multierr?status.svg
[doc]: https://godoc.org/go.uber.org/multierr
[ci-img]: https://travis-ci.org/uber-go/multierr.svg?branch=master
[cov-img]: https://codecov.io/gh/uber-go/multierr/branch/master/graph/badge.svg
[ci]: https://travis-ci.org/uber-go/multierr
[cov]: https://codecov.io/gh/uber-go/multierr
hash: b53b5e9a84b9cb3cc4b2d0499e23da2feca1eec318ce9bb717ecf35bf24bf221
updated: 2017-04-10T13:34:45.671678062-07:00
imports:
- name: go.uber.org/atomic
version: 3b8db5e93c4c02efbc313e17b2e796b0914a01fb
testImports:
- name: github.com/davecgh/go-spew
version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9
subpackages:
- spew
- name: github.com/pmezard/go-difflib
version: d8ed2627bdf02c080bf22230dbb337003b7aba2d
subpackages:
- difflib
- name: github.com/stretchr/testify
version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0
subpackages:
- assert
- require
package: go.uber.org/multierr
import:
- package: go.uber.org/atomic
version: ^1
testImport:
- package: github.com/stretchr/testify
subpackages:
- assert
coverage:
range: 80..100
round: down
precision: 2
status:
project: # measuring the overall project coverage
default: # context, you can create multiple ones with custom titles
enabled: yes # must be yes|true to enable this status
target: 95% # specify the target coverage for each commit status
# option: "auto" (must increase from parent commit or pull request base)
# option: "X%" a static target percentage to hit
if_not_found: success # if parent is not found report status as success, error, or failure
if_ci_failed: error # if ci fails report status as success, error, or failure
ignore:
- internal/readme/readme.go
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
vendor
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
*.pprof
*.out
*.log
# :zap: zap [![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Coverage Status][cov-img]][cov]
Blazing fast, structured, leveled logging in Go.
## Installation
`go get -u go.uber.org/zap`
Note that zap only supports the two most recent minor versions of Go.
## Quick Start
In contexts where performance is nice, but not critical, use the
`SugaredLogger`. It's 4-10x faster than other structured logging
packages and includes both structured and `printf`-style APIs.
```go
logger, _ := zap.NewProduction()
defer logger.Sync() // flushes buffer, if any
sugar := logger.Sugar()
sugar.Infow("failed to fetch URL",
// Structured context as loosely typed key-value pairs.
"url", url,
"attempt", 3,
"backoff", time.Second,
)
sugar.Infof("Failed to fetch URL: %s", url)
```
When performance and type safety are critical, use the `Logger`. It's even
faster than the `SugaredLogger` and allocates far less, but it only supports
structured logging.
```go
logger, _ := zap.NewProduction()
defer logger.Sync()
logger.Info("failed to fetch URL",
// Structured context as strongly typed Field values.
zap.String("url", url),
zap.Int("attempt", 3),
zap.Duration("backoff", time.Second),
)
```
See the [documentation][doc] and [FAQ](FAQ.md) for more details.
## Performance
For applications that log in the hot path, reflection-based serialization and
string formatting are prohibitively expensive &mdash; they're CPU-intensive
and make many small allocations. Put differently, using `encoding/json` and
`fmt.Fprintf` to log tons of `interface{}`s makes your application slow.
Zap takes a different approach. It includes a reflection-free, zero-allocation
JSON encoder, and the base `Logger` strives to avoid serialization overhead
and allocations wherever possible. By building the high-level `SugaredLogger`
on that foundation, zap lets users *choose* when they need to count every
allocation and when they'd prefer a more familiar, loosely typed API.
As measured by its own [benchmarking suite][], not only is zap more performant
than comparable structured logging packages &mdash; it's also faster than the
standard library. Like all benchmarks, take these with a grain of salt.<sup
id="anchor-versions">[1](#footnote-versions)</sup>
Log a message and 10 fields:
{{.BenchmarkAddingFields}}
Log a message with a logger that already has 10 fields of context:
{{.BenchmarkAccumulatedContext}}
Log a static string, without any context or `printf`-style templating:
{{.BenchmarkWithoutFields}}
## Development Status: Stable
All APIs are finalized, and no breaking changes will be made in the 1.x series
of releases. Users of semver-aware dependency management systems should pin
zap to `^1`.
## Contributing
We encourage and support an active, healthy community of contributors &mdash;
including you! Details are in the [contribution guide](CONTRIBUTING.md) and
the [code of conduct](CODE_OF_CONDUCT.md). The zap maintainers keep an eye on
issues and pull requests, but you can also report any negative conduct to
oss-conduct@uber.com. That email list is a private, safe space; even the zap
maintainers don't have access, so don't hesitate to hold us to a high
standard.
<hr>
Released under the [MIT License](LICENSE.txt).
<sup id="footnote-versions">1</sup> In particular, keep in mind that we may be
benchmarking against slightly older versions of other packages. Versions are
pinned in zap's [glide.lock][] file. [↩](#anchor-versions)
[doc-img]: https://godoc.org/go.uber.org/zap?status.svg
[doc]: https://godoc.org/go.uber.org/zap
[ci-img]: https://travis-ci.org/uber-go/zap.svg?branch=master
[ci]: https://travis-ci.org/uber-go/zap
[cov-img]: https://codecov.io/gh/uber-go/zap/branch/master/graph/badge.svg
[cov]: https://codecov.io/gh/uber-go/zap
[benchmarking suite]: https://github.com/uber-go/zap/tree/master/benchmarks
[glide.lock]: https://github.com/uber-go/zap/blob/master/glide.lock
language: go
sudo: false
go:
- 1.9.x
- 1.10.x
go_import_path: go.uber.org/zap
env:
global:
- TEST_TIMEOUT_SCALE=10
cache:
directories:
- vendor
install:
- make dependencies
script:
- make lint
- make test
- make bench
after_success:
- make cover
- bash <(curl -s https://codecov.io/bash)
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"array.go",
"config.go",
"doc.go",
"encoder.go",
"error.go",
"field.go",
"flag.go",
"global.go",
"http_handler.go",
"level.go",
"logger.go",
"options.go",
"sink.go",
"stacktrace.go",
"sugar.go",
"time.go",
"writer.go",
],
importmap = "k8s.io/kubernetes/vendor/go.uber.org/zap",
importpath = "go.uber.org/zap",
visibility = ["//visibility:public"],
deps = [
"//vendor/go.uber.org/atomic:go_default_library",
"//vendor/go.uber.org/multierr:go_default_library",
"//vendor/go.uber.org/zap/internal/bufferpool:go_default_library",
"//vendor/go.uber.org/zap/zapcore:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//vendor/go.uber.org/zap/buffer:all-srcs",
"//vendor/go.uber.org/zap/internal/bufferpool:all-srcs",
"//vendor/go.uber.org/zap/internal/color:all-srcs",
"//vendor/go.uber.org/zap/internal/exit:all-srcs",
"//vendor/go.uber.org/zap/zapcore:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age,
body size, disability, ethnicity, gender identity and expression, level of
experience, nationality, personal appearance, race, religion, or sexual
identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an
appointed representative at an online or offline event. Representation of a
project may be further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at oss-conduct@uber.com. The project
team will review and investigate all complaints, and will respond in a way
that it deems appropriate to the circumstances. The project team is obligated
to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 1.4, available at
[http://contributor-covenant.org/version/1/4][version].
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
# Contributing
We'd love your help making zap the very best structured logging library in Go!
If you'd like to add new exported APIs, please [open an issue][open-issue]
describing your proposal &mdash; discussing API changes ahead of time makes
pull request review much smoother. In your issue, pull request, and any other
communications, please remember to treat your fellow contributors with
respect! We take our [code of conduct](CODE_OF_CONDUCT.md) seriously.
Note that you'll need to sign [Uber's Contributor License Agreement][cla]
before we can accept any of your contributions. If necessary, a bot will remind
you to accept the CLA when you open your pull request.
## Setup
[Fork][fork], then clone the repository:
```
mkdir -p $GOPATH/src/go.uber.org
cd $GOPATH/src/go.uber.org
git clone git@github.com:your_github_username/zap.git
cd zap
git remote add upstream https://github.com/uber-go/zap.git
git fetch upstream
```
Install zap's dependencies:
```
make dependencies
```
Make sure that the tests and the linters pass:
```
make test
make lint
```
If you're not using the minor version of Go specified in the Makefile's
`LINTABLE_MINOR_VERSIONS` variable, `make lint` doesn't do anything. This is
fine, but it means that you'll only discover lint failures after you open your
pull request.
## Making Changes
Start by creating a new branch for your changes:
```
cd $GOPATH/src/go.uber.org/zap
git checkout master
git fetch upstream
git rebase upstream/master
git checkout -b cool_new_feature
```
Make your changes, then ensure that `make lint` and `make test` still pass. If
you're satisfied with your changes, push them to your fork.
```
git push origin cool_new_feature
```
Then use the GitHub UI to open a pull request.
At this point, you're waiting on us to review your changes. We *try* to respond
to issues and pull requests within a few business days, and we may suggest some
improvements or alternatives. Once your changes are approved, one of the
project maintainers will merge them.
We're much more likely to approve your changes if you:
* Add tests for new functionality.
* Write a [good commit message][commit-message].
* Maintain backward compatibility.
[fork]: https://github.com/uber-go/zap/fork
[open-issue]: https://github.com/uber-go/zap/issues/new
[cla]: https://cla-assistant.io/uber-go/zap
[commit-message]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
# Frequently Asked Questions
## Design
### Why spend so much effort on logger performance?
Of course, most applications won't notice the impact of a slow logger: they
already take tens or hundreds of milliseconds for each operation, so an extra
millisecond doesn't matter.
On the other hand, why *not* make structured logging fast? The `SugaredLogger`
isn't any harder to use than other logging packages, and the `Logger` makes
structured logging possible in performance-sensitive contexts. Across a fleet
of Go microservices, making each application even slightly more efficient adds
up quickly.
### Why aren't `Logger` and `SugaredLogger` interfaces?
Unlike the familiar `io.Writer` and `http.Handler`, `Logger` and
`SugaredLogger` interfaces would include *many* methods. As [Rob Pike points
out][go-proverbs], "The bigger the interface, the weaker the abstraction."
Interfaces are also rigid &mdash; *any* change requires releasing a new major
version, since it breaks all third-party implementations.
Making the `Logger` and `SugaredLogger` concrete types doesn't sacrifice much
abstraction, and it lets us add methods without introducing breaking changes.
Your applications should define and depend upon an interface that includes
just the methods you use.
### Why sample application logs?
Applications often experience runs of errors, either because of a bug or
because of a misbehaving user. Logging errors is usually a good idea, but it
can easily make this bad situation worse: not only is your application coping
with a flood of errors, it's also spending extra CPU cycles and I/O logging
those errors. Since writes are typically serialized, logging limits throughput
when you need it most.
Sampling fixes this problem by dropping repetitive log entries. Under normal
conditions, your application writes out every entry. When similar entries are
logged hundreds or thousands of times each second, though, zap begins dropping
duplicates to preserve throughput.
### Why do the structured logging APIs take a message in addition to fields?
Subjectively, we find it helpful to accompany structured context with a brief
description. This isn't critical during development, but it makes debugging
and operating unfamiliar systems much easier.
More concretely, zap's sampling algorithm uses the message to identify
duplicate entries. In our experience, this is a practical middle ground
between random sampling (which often drops the exact entry that you need while
debugging) and hashing the complete entry (which is prohibitively expensive).
### Why include package-global loggers?
Since so many other logging packages include a global logger, many
applications aren't designed to accept loggers as explicit parameters.
Changing function signatures is often a breaking change, so zap includes
global loggers to simplify migration.
Avoid them where possible.
### Why include dedicated Panic and Fatal log levels?
In general, application code should handle errors gracefully instead of using
`panic` or `os.Exit`. However, every rule has exceptions, and it's common to
crash when an error is truly unrecoverable. To avoid losing any information
&mdash; especially the reason for the crash &mdash; the logger must flush any
buffered entries before the process exits.
Zap makes this easy by offering `Panic` and `Fatal` logging methods that
automatically flush before exiting. Of course, this doesn't guarantee that
logs will never be lost, but it eliminates a common error.
See the discussion in uber-go/zap#207 for more details.
### What's `DPanic`?
`DPanic` stands for "panic in development." In development, it logs at
`PanicLevel`; otherwise, it logs at `ErrorLevel`. `DPanic` makes it easier to
catch errors that are theoretically possible, but shouldn't actually happen,
*without* crashing in production.
If you've ever written code like this, you need `DPanic`:
```go
if err != nil {
panic(fmt.Sprintf("shouldn't ever get here: %v", err))
}
```
## Installation
### What does the error `expects import "go.uber.org/zap"` mean?
Either zap was installed incorrectly or you're referencing the wrong package
name in your code.
Zap's source code happens to be hosted on GitHub, but the [import
path][import-path] is `go.uber.org/zap`. This gives us, the project
maintainers, the freedom to move the source code if necessary. However, it
means that you need to take a little care when installing and using the
package.
If you follow two simple rules, everything should work: install zap with `go
get -u go.uber.org/zap`, and always import it in your code with `import
"go.uber.org/zap"`. Your code shouldn't contain *any* references to
`github.com/uber-go/zap`.
## Usage
### Does zap support log rotation?
Zap doesn't natively support rotating log files, since we prefer to leave this
to an external program like `logrotate`.
However, it's easy to integrate a log rotation package like
[`gopkg.in/natefinch/lumberjack.v2`][lumberjack] as a `zapcore.WriteSyncer`.
```go
// lumberjack.Logger is already safe for concurrent use, so we don't need to
// lock it.
w := zapcore.AddSync(&lumberjack.Logger{
Filename: "/var/log/myapp/foo.log",
MaxSize: 500, // megabytes
MaxBackups: 3,
MaxAge: 28, // days
})
core := zapcore.NewCore(
zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()),
w,
zap.InfoLevel,
)
logger := zap.New(core)
```
## Extensions
We'd love to support every logging need within zap itself, but we're only
familiar with a handful of log ingestion systems, flag-parsing packages, and
the like. Rather than merging code that we can't effectively debug and
support, we'd rather grow an ecosystem of zap extensions.
We're aware of the following extensions, but haven't used them ourselves:
| Package | Integration |
| --- | --- |
| `github.com/tchap/zapext` | Sentry, syslog |
| `github.com/fgrosse/zaptest` | Ginkgo |
| `github.com/blendle/zapdriver` | Stackdriver |
[go-proverbs]: https://go-proverbs.github.io/
[import-path]: https://golang.org/cmd/go/#hdr-Remote_import_paths
[lumberjack]: https://godoc.org/gopkg.in/natefinch/lumberjack.v2
Copyright (c) 2016-2017 Uber Technologies, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
export GO15VENDOREXPERIMENT=1
BENCH_FLAGS ?= -cpuprofile=cpu.pprof -memprofile=mem.pprof -benchmem
PKGS ?= $(shell glide novendor)
# Many Go tools take file globs or directories as arguments instead of packages.
PKG_FILES ?= *.go zapcore benchmarks buffer zapgrpc zaptest zaptest/observer internal/bufferpool internal/exit internal/color internal/ztest
# The linting tools evolve with each Go version, so run them only on the latest
# stable release.
GO_VERSION := $(shell go version | cut -d " " -f 3)
GO_MINOR_VERSION := $(word 2,$(subst ., ,$(GO_VERSION)))
LINTABLE_MINOR_VERSIONS := 10
ifneq ($(filter $(LINTABLE_MINOR_VERSIONS),$(GO_MINOR_VERSION)),)
SHOULD_LINT := true
endif
.PHONY: all
all: lint test
.PHONY: dependencies
dependencies:
@echo "Installing Glide and locked dependencies..."
glide --version || go get -u -f github.com/Masterminds/glide
glide install
@echo "Installing test dependencies..."
go install ./vendor/github.com/axw/gocov/gocov
go install ./vendor/github.com/mattn/goveralls
ifdef SHOULD_LINT
@echo "Installing golint..."
go install ./vendor/github.com/golang/lint/golint
else
@echo "Not installing golint, since we don't expect to lint on" $(GO_VERSION)
endif
# Disable printf-like invocation checking due to testify.assert.Error()
VET_RULES := -printf=false
.PHONY: lint
lint:
ifdef SHOULD_LINT
@rm -rf lint.log
@echo "Checking formatting..."
@gofmt -d -s $(PKG_FILES) 2>&1 | tee lint.log
@echo "Installing test dependencies for vet..."
@go test -i $(PKGS)
@echo "Checking vet..."
@$(foreach dir,$(PKG_FILES),go tool vet $(VET_RULES) $(dir) 2>&1 | tee -a lint.log;)
@echo "Checking lint..."
@$(foreach dir,$(PKGS),golint $(dir) 2>&1 | tee -a lint.log;)
@echo "Checking for unresolved FIXMEs..."
@git grep -i fixme | grep -v -e vendor -e Makefile | tee -a lint.log
@echo "Checking for license headers..."
@./check_license.sh | tee -a lint.log
@[ ! -s lint.log ]
else
@echo "Skipping linters on" $(GO_VERSION)
endif
.PHONY: test
test:
go test -race $(PKGS)
.PHONY: cover
cover:
./scripts/cover.sh $(PKGS)
.PHONY: bench
BENCH ?= .
bench:
@$(foreach pkg,$(PKGS),go test -bench=$(BENCH) -run="^$$" $(BENCH_FLAGS) $(pkg);)
.PHONY: updatereadme
updatereadme:
rm -f README.md
cat .readme.tmpl | go run internal/readme/readme.go > README.md
# :zap: zap [![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Coverage Status][cov-img]][cov]
Blazing fast, structured, leveled logging in Go.
## Installation
`go get -u go.uber.org/zap`
Note that zap only supports the two most recent minor versions of Go.
## Quick Start
In contexts where performance is nice, but not critical, use the
`SugaredLogger`. It's 4-10x faster than other structured logging
packages and includes both structured and `printf`-style APIs.
```go
logger, _ := zap.NewProduction()
defer logger.Sync() // flushes buffer, if any
sugar := logger.Sugar()
sugar.Infow("failed to fetch URL",
// Structured context as loosely typed key-value pairs.
"url", url,
"attempt", 3,
"backoff", time.Second,
)
sugar.Infof("Failed to fetch URL: %s", url)
```
When performance and type safety are critical, use the `Logger`. It's even
faster than the `SugaredLogger` and allocates far less, but it only supports
structured logging.
```go
logger, _ := zap.NewProduction()
defer logger.Sync()
logger.Info("failed to fetch URL",
// Structured context as strongly typed Field values.
zap.String("url", url),
zap.Int("attempt", 3),
zap.Duration("backoff", time.Second),
)
```
See the [documentation][doc] and [FAQ](FAQ.md) for more details.
## Performance
For applications that log in the hot path, reflection-based serialization and
string formatting are prohibitively expensive &mdash; they're CPU-intensive
and make many small allocations. Put differently, using `encoding/json` and
`fmt.Fprintf` to log tons of `interface{}`s makes your application slow.
Zap takes a different approach. It includes a reflection-free, zero-allocation
JSON encoder, and the base `Logger` strives to avoid serialization overhead
and allocations wherever possible. By building the high-level `SugaredLogger`
on that foundation, zap lets users *choose* when they need to count every
allocation and when they'd prefer a more familiar, loosely typed API.
As measured by its own [benchmarking suite][], not only is zap more performant
than comparable structured logging packages &mdash; it's also faster than the
standard library. Like all benchmarks, take these with a grain of salt.<sup
id="anchor-versions">[1](#footnote-versions)</sup>
Log a message and 10 fields:
| Package | Time | Objects Allocated |
| :--- | :---: | :---: |
| :zap: zap | 3131 ns/op | 5 allocs/op |
| :zap: zap (sugared) | 4173 ns/op | 21 allocs/op |
| zerolog | 16154 ns/op | 90 allocs/op |
| lion | 16341 ns/op | 111 allocs/op |
| go-kit | 17049 ns/op | 126 allocs/op |
| logrus | 23662 ns/op | 142 allocs/op |
| log15 | 36351 ns/op | 149 allocs/op |
| apex/log | 42530 ns/op | 126 allocs/op |
Log a message with a logger that already has 10 fields of context:
| Package | Time | Objects Allocated |
| :--- | :---: | :---: |
| :zap: zap | 380 ns/op | 0 allocs/op |
| :zap: zap (sugared) | 564 ns/op | 2 allocs/op |
| zerolog | 321 ns/op | 0 allocs/op |
| lion | 7092 ns/op | 39 allocs/op |
| go-kit | 20226 ns/op | 115 allocs/op |
| logrus | 22312 ns/op | 130 allocs/op |
| log15 | 28788 ns/op | 79 allocs/op |
| apex/log | 42063 ns/op | 115 allocs/op |
Log a static string, without any context or `printf`-style templating:
| Package | Time | Objects Allocated |
| :--- | :---: | :---: |
| :zap: zap | 361 ns/op | 0 allocs/op |
| :zap: zap (sugared) | 534 ns/op | 2 allocs/op |
| zerolog | 323 ns/op | 0 allocs/op |
| standard library | 575 ns/op | 2 allocs/op |
| go-kit | 922 ns/op | 13 allocs/op |
| lion | 1413 ns/op | 10 allocs/op |
| logrus | 2291 ns/op | 27 allocs/op |
| apex/log | 3690 ns/op | 11 allocs/op |
| log15 | 5954 ns/op | 26 allocs/op |
## Development Status: Stable
All APIs are finalized, and no breaking changes will be made in the 1.x series
of releases. Users of semver-aware dependency management systems should pin
zap to `^1`.
## Contributing
We encourage and support an active, healthy community of contributors &mdash;
including you! Details are in the [contribution guide](CONTRIBUTING.md) and
the [code of conduct](CODE_OF_CONDUCT.md). The zap maintainers keep an eye on
issues and pull requests, but you can also report any negative conduct to
oss-conduct@uber.com. That email list is a private, safe space; even the zap
maintainers don't have access, so don't hesitate to hold us to a high
standard.
<hr>
Released under the [MIT License](LICENSE.txt).
<sup id="footnote-versions">1</sup> In particular, keep in mind that we may be
benchmarking against slightly older versions of other packages. Versions are
pinned in zap's [glide.lock][] file. [](#anchor-versions)
[doc-img]: https://godoc.org/go.uber.org/zap?status.svg
[doc]: https://godoc.org/go.uber.org/zap
[ci-img]: https://travis-ci.org/uber-go/zap.svg?branch=master
[ci]: https://travis-ci.org/uber-go/zap
[cov-img]: https://codecov.io/gh/uber-go/zap/branch/master/graph/badge.svg
[cov]: https://codecov.io/gh/uber-go/zap
[benchmarking suite]: https://github.com/uber-go/zap/tree/master/benchmarks
[glide.lock]: https://github.com/uber-go/zap/blob/master/glide.lock
// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package zap
import (
"time"
"go.uber.org/zap/zapcore"
)
// Array constructs a field with the given key and ArrayMarshaler. It provides
// a flexible, but still type-safe and efficient, way to add array-like types
// to the logging context. The struct's MarshalLogArray method is called lazily.
func Array(key string, val zapcore.ArrayMarshaler) Field {
return Field{Key: key, Type: zapcore.ArrayMarshalerType, Interface: val}
}
// Bools constructs a field that carries a slice of bools.
func Bools(key string, bs []bool) Field {
return Array(key, bools(bs))
}
// ByteStrings constructs a field that carries a slice of []byte, each of which
// must be UTF-8 encoded text.
func ByteStrings(key string, bss [][]byte) Field {
return Array(key, byteStringsArray(bss))
}
// Complex128s constructs a field that carries a slice of complex numbers.
func Complex128s(key string, nums []complex128) Field {
return Array(key, complex128s(nums))
}
// Complex64s constructs a field that carries a slice of complex numbers.
func Complex64s(key string, nums []complex64) Field {
return Array(key, complex64s(nums))
}
// Durations constructs a field that carries a slice of time.Durations.
func Durations(key string, ds []time.Duration) Field {
return Array(key, durations(ds))
}
// Float64s constructs a field that carries a slice of floats.
func Float64s(key string, nums []float64) Field {
return Array(key, float64s(nums))
}
// Float32s constructs a field that carries a slice of floats.
func Float32s(key string, nums []float32) Field {
return Array(key, float32s(nums))
}
// Ints constructs a field that carries a slice of integers.
func Ints(key string, nums []int) Field {
return Array(key, ints(nums))
}
// Int64s constructs a field that carries a slice of integers.
func Int64s(key string, nums []int64) Field {
return Array(key, int64s(nums))
}
// Int32s constructs a field that carries a slice of integers.
func Int32s(key string, nums []int32) Field {
return Array(key, int32s(nums))
}
// Int16s constructs a field that carries a slice of integers.
func Int16s(key string, nums []int16) Field {
return Array(key, int16s(nums))
}
// Int8s constructs a field that carries a slice of integers.
func Int8s(key string, nums []int8) Field {
return Array(key, int8s(nums))
}
// Strings constructs a field that carries a slice of strings.
func Strings(key string, ss []string) Field {
return Array(key, stringArray(ss))
}
// Times constructs a field that carries a slice of time.Times.
func Times(key string, ts []time.Time) Field {
return Array(key, times(ts))
}
// Uints constructs a field that carries a slice of unsigned integers.
func Uints(key string, nums []uint) Field {
return Array(key, uints(nums))
}
// Uint64s constructs a field that carries a slice of unsigned integers.
func Uint64s(key string, nums []uint64) Field {
return Array(key, uint64s(nums))
}
// Uint32s constructs a field that carries a slice of unsigned integers.
func Uint32s(key string, nums []uint32) Field {
return Array(key, uint32s(nums))
}
// Uint16s constructs a field that carries a slice of unsigned integers.
func Uint16s(key string, nums []uint16) Field {
return Array(key, uint16s(nums))
}
// Uint8s constructs a field that carries a slice of unsigned integers.
func Uint8s(key string, nums []uint8) Field {
return Array(key, uint8s(nums))
}
// Uintptrs constructs a field that carries a slice of pointer addresses.
func Uintptrs(key string, us []uintptr) Field {
return Array(key, uintptrs(us))
}
// Errors constructs a field that carries a slice of errors.
func Errors(key string, errs []error) Field {
return Array(key, errArray(errs))
}
type bools []bool
func (bs bools) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range bs {
arr.AppendBool(bs[i])
}
return nil
}
type byteStringsArray [][]byte
func (bss byteStringsArray) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range bss {
arr.AppendByteString(bss[i])
}
return nil
}
type complex128s []complex128
func (nums complex128s) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendComplex128(nums[i])
}
return nil
}
type complex64s []complex64
func (nums complex64s) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendComplex64(nums[i])
}
return nil
}
type durations []time.Duration
func (ds durations) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range ds {
arr.AppendDuration(ds[i])
}
return nil
}
type float64s []float64
func (nums float64s) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendFloat64(nums[i])
}
return nil
}
type float32s []float32
func (nums float32s) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendFloat32(nums[i])
}
return nil
}
type ints []int
func (nums ints) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendInt(nums[i])
}
return nil
}
type int64s []int64
func (nums int64s) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendInt64(nums[i])
}
return nil
}
type int32s []int32
func (nums int32s) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendInt32(nums[i])
}
return nil
}
type int16s []int16
func (nums int16s) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendInt16(nums[i])
}
return nil
}
type int8s []int8
func (nums int8s) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendInt8(nums[i])
}
return nil
}
type stringArray []string
func (ss stringArray) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range ss {
arr.AppendString(ss[i])
}
return nil
}
type times []time.Time
func (ts times) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range ts {
arr.AppendTime(ts[i])
}
return nil
}
type uints []uint
func (nums uints) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendUint(nums[i])
}
return nil
}
type uint64s []uint64
func (nums uint64s) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendUint64(nums[i])
}
return nil
}
type uint32s []uint32
func (nums uint32s) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendUint32(nums[i])
}
return nil
}
type uint16s []uint16
func (nums uint16s) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendUint16(nums[i])
}
return nil
}
type uint8s []uint8
func (nums uint8s) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendUint8(nums[i])
}
return nil
}
type uintptrs []uintptr
func (nums uintptrs) MarshalLogArray(arr zapcore.ArrayEncoder) error {
for i := range nums {
arr.AppendUintptr(nums[i])
}
return nil
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment