OpenTelemetry support

.golangci.yml

@@ -1,6 +1,7 @@
 run:
     skip-dirs:
         - .gen
+        - pkg/kitx/tracing/opentelemetry
 
     skip-files:
         - ".*_gen\\.go$"

go.mod

@@ -33,6 +33,7 @@ require (
 	github.com/stretchr/testify v1.5.1
 	github.com/vektah/gqlparser/v2 v2.0.1
 	go.opencensus.io v0.22.3
+	go.opentelemetry.io/otel v0.4.3
 	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
 	google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d
 	google.golang.org/grpc v1.28.1

go.sum

@@ -21,6 +21,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
+github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7 h1:qELHH0AWCvf98Yf+CNIJx9vOZOfHFDDzgDRYsnNk/vs=
+github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
@@ -51,6 +53,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
+github.com/benbjohnson/clock v1.0.0 h1:78Jk/r6m4wCi6sndMpty7A//t4dw/RW5fV4ZgDVfX1w=
+github.com/benbjohnson/clock v1.0.0/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
@@ -172,6 +176,9 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -334,6 +341,7 @@ github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA=
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
@@ -494,6 +502,8 @@ go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opentelemetry.io/otel v0.4.3 h1:CroUX/0O1ZDcF0iWOO8gwYFWb5EbdSF0/C1yosO+Vhs=
+go.opentelemetry.io/otel v0.4.3/go.mod h1:jzBIgIzK43Iu1BpDAXwqOd6UPsSAk+ewVZ5ofSXw4Ek=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -654,6 +664,7 @@ google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610 h1:Ygq9/SRJX9+dU0W
 google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d h1:I7Vuu5Ejagca+VcgfBINHke3xwjCTYnIG4Q57fv0wYY=
 google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=

pkg/kitx/tracing/opentelemetry/doc.go

@@ -0,0 +1,4 @@
+// Package opentelemetry provides Go kit integration to the OpenTelemetry project.
+//
+// OpenTelemetry makes robust, portable telemetry a built-in feature of cloud-native software.
+package opentelemetry

pkg/kitx/tracing/opentelemetry/endpoint.go

@@ -0,0 +1,105 @@
+package opentelemetry
+
+import (
+	"context"
+	"strconv"
+
+	"go.opentelemetry.io/otel/api/core"
+	"go.opentelemetry.io/otel/api/global"
+	"go.opentelemetry.io/otel/api/key"
+	"go.opentelemetry.io/otel/api/trace"
+	"google.golang.org/grpc/codes"
+
+	"github.com/go-kit/kit/endpoint"
+	"github.com/go-kit/kit/sd/lb"
+)
+
+// TraceEndpointDefaultName is the default endpoint span name to use.
+const TraceEndpointDefaultName = "gokit/endpoint"
+
+// TraceEndpoint returns an Endpoint middleware, tracing a Go kit endpoint.
+// This endpoint tracer should be used in combination with a Go kit Transport
+// tracing middleware, generic OpenTelemetry transport middleware or custom before
+// and after transport functions as service propagation of SpanContext is not
+// provided in this middleware.
+func TraceEndpoint(options ...EndpointOption) endpoint.Middleware {
+	cfg := &EndpointOptions{}
+
+	global.Tracer("")
+
+	for _, o := range options {
+		o(cfg)
+	}
+
+	if cfg.Tracer == nil {
+		cfg.Tracer = global.Tracer("")
+	}
+
+	if cfg.DefaultName == "" {
+		cfg.DefaultName = TraceEndpointDefaultName
+	}
+
+	return func(next endpoint.Endpoint) endpoint.Endpoint {
+		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
+			name := cfg.DefaultName
+
+			if cfg.GetName != nil {
+				if newName := cfg.GetName(ctx, name); newName != "" {
+					name = newName
+				}
+			}
+
+			ctx, span := cfg.Tracer.Start(
+				ctx, name,
+				trace.WithAttributes(cfg.Attributes...),
+				trace.WithAttributes(cfg.getAttributes(ctx)...),
+			)
+			defer span.End()
+
+			defer func() {
+				if err != nil {
+					if lberr, ok := err.(lb.RetryError); ok {
+						// handle errors originating from lb.Retry
+						attrs := make([]core.KeyValue, 0, len(lberr.RawErrors))
+						for idx, rawErr := range lberr.RawErrors {
+							attrs = append(attrs, key.String("gokit.retry.error."+strconv.Itoa(idx+1), rawErr.Error()))
+						}
+
+						span.SetAttributes(attrs...)
+						span.SetStatus(codes.Unknown, lberr.Final.Error())
+
+						return
+					}
+
+					// generic error
+					span.SetStatus(codes.Unknown, err.Error())
+
+					return
+				}
+
+				// test for business error
+				if res, ok := response.(endpoint.Failer); ok && res.Failed() != nil {
+					span.SetAttributes(key.String("gokit.business.error", res.Failed().Error()))
+
+					if cfg.IgnoreBusinessError {
+						// status ok
+
+						return
+					}
+
+					// treating business error as real error in span.
+					span.SetStatus(codes.Unknown, res.Failed().Error())
+
+					return
+				}
+
+				// no errors identified
+				// status ok
+			}()
+
+			response, err = next(ctx, request)
+
+			return
+		}
+	}
+}

pkg/kitx/tracing/opentelemetry/endpoint_options.go

@@ -0,0 +1,102 @@
+package opentelemetry
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel/api/core"
+	"go.opentelemetry.io/otel/api/trace"
+)
+
+// EndpointOptions holds the options for tracing an endpoint
+type EndpointOptions struct {
+	// Tracer (if specified) is used for starting new spans.
+	// Falls back to a global tracer with an empty name.
+	//
+	// See https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/api.md#obtaining-a-tracer
+	Tracer trace.Tracer
+
+	// DefaultName is used as a fallback if GetName is not specified.
+	DefaultName string
+
+	// IgnoreBusinessError if set to true will not treat a business error
+	// identified through the endpoint.Failer interface as a span error.
+	IgnoreBusinessError bool
+
+	// Attributes holds the default attributes which will be set on span
+	// creation by our Endpoint middleware.
+	Attributes []core.KeyValue
+
+	// GetName is an optional function that can set the span name based on the existing name
+	// for the endpoint and information in the context.
+	//
+	// If the function is nil, or the returned name is empty, the existing name for the endpoint is used.
+	GetName func(ctx context.Context, name string) string
+
+	// GetAttributes is an optional function that can extract trace attributes
+	// from the context and add them to the span.
+	GetAttributes func(ctx context.Context) []core.KeyValue
+}
+
+func (o EndpointOptions) getAttributes(ctx context.Context) []core.KeyValue {
+	if o.GetAttributes == nil {
+		return nil
+	}
+
+	return o.GetAttributes(ctx)
+}
+
+// EndpointOption allows for functional options to our OpenTelemetry endpoint
+// tracing middleware.
+type EndpointOption func(*EndpointOptions)
+
+// WithEndpointConfig sets all configuration options at once by use of the
+// EndpointOptions struct.
+func WithEndpointConfig(options EndpointOptions) EndpointOption {
+	return func(o *EndpointOptions) {
+		*o = options
+	}
+}
+
+// WithTracer sets the tracer.
+func WithTracer(tracer trace.Tracer) EndpointOption {
+	return func(o *EndpointOptions) {
+		o.Tracer = tracer
+	}
+}
+
+// WithDefaultName sets the default name.
+func WithDefaultName(defaultName string) EndpointOption {
+	return func(o *EndpointOptions) {
+		o.DefaultName = defaultName
+	}
+}
+
+// WithEndpointAttributes sets the default attributes for the spans created by
+// the Endpoint tracer.
+func WithEndpointAttributes(attrs ...core.KeyValue) EndpointOption {
+	return func(o *EndpointOptions) {
+		o.Attributes = attrs
+	}
+}
+
+// WithIgnoreBusinessError if set to true will not treat a business error
+// identified through the endpoint.Failer interface as a span error.
+func WithIgnoreBusinessError(val bool) EndpointOption {
+	return func(o *EndpointOptions) {
+		o.IgnoreBusinessError = val
+	}
+}
+
+// WithSpanName extracts additional attributes from the request context.
+func WithSpanName(fn func(ctx context.Context, name string) string) EndpointOption {
+	return func(o *EndpointOptions) {
+		o.GetName = fn
+	}
+}
+
+// WithSpanAttributes extracts additional attributes from the request context.
+func WithSpanAttributes(fn func(ctx context.Context) []core.KeyValue) EndpointOption {
+	return func(o *EndpointOptions) {
+		o.GetAttributes = fn
+	}
+}