tracing: New OpenTelemetry module (#4361)

* opentelemetry: create a new module

* fix imports

* fix test

* Update modules/caddyhttp/opentelemetry/README.md

Co-authored-by: Dave Henderson <[email protected]>

* Update modules/caddyhttp/opentelemetry/README.md

Co-authored-by: Dave Henderson <[email protected]>

* Update modules/caddyhttp/opentelemetry/README.md

Co-authored-by: Dave Henderson <[email protected]>

* Update modules/caddyhttp/opentelemetry/tracer.go

Co-authored-by: Dave Henderson <[email protected]>

* rename error ErrUnsupportedTracesProtocol

* replace spaces with tabs in the test data

* Update modules/caddyhttp/opentelemetry/README.md

Co-authored-by: Francis Lavoie <[email protected]>

* Update modules/caddyhttp/opentelemetry/README.md

Co-authored-by: Francis Lavoie <[email protected]>

* replace spaces with tabs in the README.md

* use default values for a propagation and exporter protocol

* set http attributes with helper

* simplify code

* Cleanup modules/caddyhttp/opentelemetry/README.md

Co-authored-by: Dave Henderson <[email protected]>

* Update link in README.md

Co-authored-by: Dave Henderson <[email protected]>

* Update documentation in README.md

Co-authored-by: Dave Henderson <[email protected]>

* Update link to naming spec in README.md

Co-authored-by: Dave Henderson <[email protected]>

* Rename module from opentelemetry to tracing

Co-authored-by: Dave Henderson <[email protected]>

* Rename span_name to span

Co-authored-by: Dave Henderson <[email protected]>

* Rename span_name to span

Co-authored-by: Dave Henderson <[email protected]>

* Simplify otel resource creation

Co-authored-by: Dave Henderson <[email protected]>

* handle extra attributes

Co-authored-by: Dave Henderson <[email protected]>

* update go.opentelemetry.io/otel/semconv to 1.7.0

Co-authored-by: Dave Henderson <[email protected]>

* update go.opentelemetry.io/otel version

* remove environment variable handling

* always use tracecontext,baggage as propagators

* extract tracer name into variable

* rename OpenTelemetry to Tracing

* simplify resource creation

* update go.mod

* rename package from opentelemetry to tracing

* cleanup tests

* update Caddyfile example in README.md

* update README.md

* fix test

* fix module name in README.md

* fix module name in README.md

* change names in README.md and tests

* order imports

* remove redundant tests

* Update documentation README.md

Co-authored-by: Dave Henderson <[email protected]>

* Fix grammar

Co-authored-by: Dave Henderson <[email protected]>

* Update comments

Co-authored-by: Dave Henderson <[email protected]>

* Update comments

Co-authored-by: Dave Henderson <[email protected]>

* update go.sum

* update go.sum

* Add otelhttp instrumentation, update OpenTelemetry libraries.

* Use otelhttp instrumentation for instrumenting HTTP requests.

This change uses context.WithValue to inject the next handler into the
request context via a "nextCall" carrier struct, and pass it on to a
standard Go HTTP handler returned by otelhttp.NewHandler. The
underlying handler will extract the next handler from the context,
call it and pass the returned error to the carrier struct.

* use zap.Error() for the error log

* remove README.md

* update dependencies

* clean up the code

* change comment

* move serveHTTP method from separate file

* add syntax to the UnmarshalCaddyfile comment

* go import the file

* admin: Write proper status on invalid requests (#4569) (fix #4561)

* update dependencies

Co-authored-by: Dave Henderson <[email protected]>
Co-authored-by: Francis Lavoie <[email protected]>
Co-authored-by: Vibhav Pant <[email protected]>
Co-authored-by: Alok Naushad <[email protected]>
Co-authored-by: Cedric Ziel <[email protected]>

6 files changed, 544 insertions, 0 deletions

diff --git a/modules/caddyhttp/tracing/module.go b/modules/caddyhttp/tracing/module.go
new file mode 100644
index 000000000..7cce6692b
--- /dev/null
+++ b/modules/caddyhttp/tracing/module.go
@@ -0,0 +1,121 @@
+package tracing
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"go.uber.org/zap"
+)
+
+func init() {
+	caddy.RegisterModule(Tracing{})
+	httpcaddyfile.RegisterHandlerDirective("tracing", parseCaddyfile)
+}
+
+// Tracing implements an HTTP handler that adds support for distributed tracing,
+// using OpenTelemetry. This module is responsible for the injection and
+// propagation of the trace context. Configure this module via environment
+// variables (see https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md).
+// Some values can be overwritten in the configuration file.
+type Tracing struct {
+	// SpanName is a span name. It should follow the naming guidelines here:
+	// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#span
+	SpanName string `json:"span"`
+
+	// otel implements opentelemetry related logic.
+	otel openTelemetryWrapper
+
+	logger *zap.Logger
+}
+
+// CaddyModule returns the Caddy module information.
+func (Tracing) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.handlers.tracing",
+		New: func() caddy.Module { return new(Tracing) },
+	}
+}
+
+// Provision implements caddy.Provisioner.
+func (ot *Tracing) Provision(ctx caddy.Context) error {
+	ot.logger = ctx.Logger(ot)
+
+	var err error
+	ot.otel, err = newOpenTelemetryWrapper(ctx, ot.SpanName)
+
+	return err
+}
+
+// Cleanup implements caddy.CleanerUpper and closes any idle connections. It
+// calls Shutdown method for a trace provider https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#shutdown.
+func (ot *Tracing) Cleanup() error {
+	if err := ot.otel.cleanup(ot.logger); err != nil {
+		return fmt.Errorf("tracerProvider shutdown: %w", err)
+	}
+	return nil
+}
+
+// ServeHTTP implements caddyhttp.MiddlewareHandler.
+func (ot *Tracing) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
+	return ot.otel.ServeHTTP(w, r, next)
+}
+
+// UnmarshalCaddyfile sets up the module from Caddyfile tokens. Syntax:
+//
+//     tracing {
+//         [span <span_name>]
+//     }
+//
+func (ot *Tracing) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	setParameter := func(d *caddyfile.Dispenser, val *string) error {
+		if d.NextArg() {
+			*val = d.Val()
+		} else {
+			return d.ArgErr()
+		}
+		if d.NextArg() {
+			return d.ArgErr()
+		}
+		return nil
+	}
+
+	// paramsMap is a mapping between "string" parameter from the Caddyfile and its destination within the module
+	paramsMap := map[string]*string{
+		"span": &ot.SpanName,
+	}
+
+	for d.Next() {
+		args := d.RemainingArgs()
+		if len(args) > 0 {
+			return d.ArgErr()
+		}
+
+		for d.NextBlock(0) {
+			if dst, ok := paramsMap[d.Val()]; ok {
+				if err := setParameter(d, dst); err != nil {
+					return err
+				}
+			} else {
+				return d.ArgErr()
+			}
+		}
+	}
+	return nil
+}
+
+func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
+	var m Tracing
+	err := m.UnmarshalCaddyfile(h.Dispenser)
+	return &m, err
+}
+
+// Interface guards
+var (
+	_ caddy.Provisioner           = (*Tracing)(nil)
+	_ caddyhttp.MiddlewareHandler = (*Tracing)(nil)
+	_ caddyfile.Unmarshaler       = (*Tracing)(nil)
+)
diff --git a/modules/caddyhttp/tracing/module_test.go b/modules/caddyhttp/tracing/module_test.go
new file mode 100644
index 000000000..0fbc05bbd
--- /dev/null
+++ b/modules/caddyhttp/tracing/module_test.go
@@ -0,0 +1,182 @@
+package tracing
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+)
+
+func TestTracing_UnmarshalCaddyfile(t *testing.T) {
+	tests := []struct {
+		name     string
+		spanName string
+		d        *caddyfile.Dispenser
+		wantErr  bool
+	}{
+		{
+			name:     "Full config",
+			spanName: "my-span",
+			d: caddyfile.NewTestDispenser(`
+tracing {
+	span my-span
+}`),
+			wantErr: false,
+		},
+		{
+			name:     "Only span name in the config",
+			spanName: "my-span",
+			d: caddyfile.NewTestDispenser(`
+tracing {
+	span my-span
+}`),
+			wantErr: false,
+		},
+		{
+			name: "Empty config",
+			d: caddyfile.NewTestDispenser(`
+tracing {
+}`),
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ot := &Tracing{}
+			if err := ot.UnmarshalCaddyfile(tt.d); (err != nil) != tt.wantErr {
+				t.Errorf("UnmarshalCaddyfile() error = %v, wantErrType %v", err, tt.wantErr)
+			}
+
+			if ot.SpanName != tt.spanName {
+				t.Errorf("UnmarshalCaddyfile() SpanName = %v, want SpanName %v", ot.SpanName, tt.spanName)
+			}
+		})
+	}
+}
+
+func TestTracing_UnmarshalCaddyfile_Error(t *testing.T) {
+	tests := []struct {
+		name    string
+		d       *caddyfile.Dispenser
+		wantErr bool
+	}{
+		{
+			name: "Unknown parameter",
+			d: caddyfile.NewTestDispenser(`
+		tracing {
+			foo bar
+		}`),
+			wantErr: true,
+		},
+		{
+			name: "Missed argument",
+			d: caddyfile.NewTestDispenser(`
+tracing {
+	span
+}`),
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ot := &Tracing{}
+			if err := ot.UnmarshalCaddyfile(tt.d); (err != nil) != tt.wantErr {
+				t.Errorf("UnmarshalCaddyfile() error = %v, wantErrType %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestTracing_ServeHTTP_Propagation_Without_Initial_Headers(t *testing.T) {
+	ot := &Tracing{
+		SpanName: "mySpan",
+	}
+
+	req := httptest.NewRequest("GET", "https://example.com/foo", nil)
+	w := httptest.NewRecorder()
+
+	var handler caddyhttp.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) error {
+		traceparent := request.Header.Get("Traceparent")
+		if traceparent == "" || strings.HasPrefix(traceparent, "00-00000000000000000000000000000000-0000000000000000") {
+			t.Errorf("Invalid traceparent: %v", traceparent)
+		}
+
+		return nil
+	}
+
+	ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
+	defer cancel()
+
+	if err := ot.Provision(ctx); err != nil {
+		t.Errorf("Provision error: %v", err)
+		t.FailNow()
+	}
+
+	if err := ot.ServeHTTP(w, req, handler); err != nil {
+		t.Errorf("ServeHTTP error: %v", err)
+	}
+}
+
+func TestTracing_ServeHTTP_Propagation_With_Initial_Headers(t *testing.T) {
+	ot := &Tracing{
+		SpanName: "mySpan",
+	}
+
+	req := httptest.NewRequest("GET", "https://example.com/foo", nil)
+	req.Header.Set("traceparent", "00-11111111111111111111111111111111-1111111111111111-01")
+	w := httptest.NewRecorder()
+
+	var handler caddyhttp.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) error {
+		traceparent := request.Header.Get("Traceparent")
+		if !strings.HasPrefix(traceparent, "00-11111111111111111111111111111111") {
+			t.Errorf("Invalid traceparent: %v", traceparent)
+		}
+
+		return nil
+	}
+
+	ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
+	defer cancel()
+
+	if err := ot.Provision(ctx); err != nil {
+		t.Errorf("Provision error: %v", err)
+		t.FailNow()
+	}
+
+	if err := ot.ServeHTTP(w, req, handler); err != nil {
+		t.Errorf("ServeHTTP error: %v", err)
+	}
+}
+
+func TestTracing_ServeHTTP_Next_Error(t *testing.T) {
+	ot := &Tracing{
+		SpanName: "mySpan",
+	}
+
+	req := httptest.NewRequest("GET", "https://example.com/foo", nil)
+	w := httptest.NewRecorder()
+
+	expectErr := errors.New("test error")
+
+	var handler caddyhttp.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) error {
+		return expectErr
+	}
+
+	ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
+	defer cancel()
+
+	if err := ot.Provision(ctx); err != nil {
+		t.Errorf("Provision error: %v", err)
+		t.FailNow()
+	}
+
+	if err := ot.ServeHTTP(w, req, handler); err == nil || !errors.Is(err, expectErr) {
+		t.Errorf("expected error, got: %v", err)
+	}
+}
diff --git a/modules/caddyhttp/tracing/tracer.go b/modules/caddyhttp/tracing/tracer.go
new file mode 100644
index 000000000..ce23944cf
--- /dev/null
+++ b/modules/caddyhttp/tracing/tracer.go
@@ -0,0 +1,108 @@
+package tracing
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/caddyserver/caddy/v2"
+
+	caddycmd "github.com/caddyserver/caddy/v2/cmd"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
+	"go.uber.org/zap"
+)
+
+const (
+	webEngineName                = "Caddy"
+	defaultSpanName              = "handler"
+	nextCallCtxKey  caddy.CtxKey = "nextCall"
+)
+
+// nextCall store the next handler, and the error value return on calling it (if any)
+type nextCall struct {
+	next caddyhttp.Handler
+	err  error
+}
+
+// openTelemetryWrapper is responsible for the tracing injection, extraction and propagation.
+type openTelemetryWrapper struct {
+	propagators propagation.TextMapPropagator
+
+	handler http.Handler
+
+	spanName string
+}
+
+// newOpenTelemetryWrapper is responsible for the openTelemetryWrapper initialization using provided configuration.
+func newOpenTelemetryWrapper(
+	ctx context.Context,
+	spanName string,
+) (openTelemetryWrapper, error) {
+	if spanName == "" {
+		spanName = defaultSpanName
+	}
+
+	ot := openTelemetryWrapper{
+		spanName: spanName,
+	}
+
+	res, err := ot.newResource(webEngineName, caddycmd.CaddyVersion())
+	if err != nil {
+		return ot, fmt.Errorf("creating resource error: %w", err)
+	}
+
+	traceExporter, err := otlptracegrpc.New(ctx)
+	if err != nil {
+		return ot, fmt.Errorf("creating trace exporter error: %w", err)
+	}
+
+	ot.propagators = propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
+
+	tracerProvider := globalTracerProvider.getTracerProvider(
+		sdktrace.WithBatcher(traceExporter),
+		sdktrace.WithResource(res),
+	)
+
+	ot.handler = otelhttp.NewHandler(http.HandlerFunc(ot.serveHTTP), ot.spanName, otelhttp.WithTracerProvider(tracerProvider), otelhttp.WithPropagators(ot.propagators))
+	return ot, nil
+}
+
+// serveHTTP injects a tracing context and call the next handler.
+func (ot *openTelemetryWrapper) serveHTTP(w http.ResponseWriter, r *http.Request) {
+	ot.propagators.Inject(r.Context(), propagation.HeaderCarrier(r.Header))
+	next := r.Context().Value(nextCallCtxKey).(*nextCall)
+	next.err = next.next.ServeHTTP(w, r)
+}
+
+// ServeHTTP propagates call to the by wrapped by `otelhttp` next handler.
+func (ot *openTelemetryWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
+	n := &nextCall{
+		next: next,
+		err:  nil,
+	}
+	ot.handler.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), nextCallCtxKey, n)))
+
+	return n.err
+}
+
+// cleanup flush all remaining data and shutdown a tracerProvider
+func (ot *openTelemetryWrapper) cleanup(logger *zap.Logger) error {
+	return globalTracerProvider.cleanupTracerProvider(logger)
+}
+
+// newResource creates a resource that describe current handler instance and merge it with a default attributes value.
+func (ot *openTelemetryWrapper) newResource(
+	webEngineName,
+	webEngineVersion string,
+) (*resource.Resource, error) {
+	return resource.Merge(resource.Default(), resource.NewSchemaless(
+		semconv.WebEngineNameKey.String(webEngineName),
+		semconv.WebEngineVersionKey.String(webEngineVersion),
+	))
+}
diff --git a/modules/caddyhttp/tracing/tracer_test.go b/modules/caddyhttp/tracing/tracer_test.go
new file mode 100644
index 000000000..36a32ff46
--- /dev/null
+++ b/modules/caddyhttp/tracing/tracer_test.go
@@ -0,0 +1,27 @@
+package tracing
+
+import (
+	"context"
+	"testing"
+
+	"github.com/caddyserver/caddy/v2"
+)
+
+func TestOpenTelemetryWrapper_newOpenTelemetryWrapper(t *testing.T) {
+	ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
+	defer cancel()
+
+	var otw openTelemetryWrapper
+	var err error
+
+	if otw, err = newOpenTelemetryWrapper(ctx,
+		"",
+	); err != nil {
+		t.Errorf("newOpenTelemetryWrapper() error = %v", err)
+		t.FailNow()
+	}
+
+	if otw.propagators == nil {
+		t.Errorf("Propagators should not be empty")
+	}
+}
diff --git a/modules/caddyhttp/tracing/tracerprovider.go b/modules/caddyhttp/tracing/tracerprovider.go
new file mode 100644
index 000000000..035425ed6
--- /dev/null
+++ b/modules/caddyhttp/tracing/tracerprovider.go
@@ -0,0 +1,63 @@
+package tracing
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.uber.org/zap"
+)
+
+// globalTracerProvider stores global tracer provider and is responsible for graceful shutdown when nobody is using it.
+var globalTracerProvider = &tracerProvider{}
+
+type tracerProvider struct {
+	mu                     sync.Mutex
+	tracerProvider         *sdktrace.TracerProvider
+	tracerProvidersCounter int
+}
+
+// getTracerProvider create or return an existing global TracerProvider
+func (t *tracerProvider) getTracerProvider(opts ...sdktrace.TracerProviderOption) *sdktrace.TracerProvider {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	t.tracerProvidersCounter++
+
+	if t.tracerProvider == nil {
+		t.tracerProvider = sdktrace.NewTracerProvider(
+			opts...,
+		)
+	}
+
+	return t.tracerProvider
+}
+
+// cleanupTracerProvider gracefully shutdown a TracerProvider
+func (t *tracerProvider) cleanupTracerProvider(logger *zap.Logger) error {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	if t.tracerProvidersCounter > 0 {
+		t.tracerProvidersCounter--
+	}
+
+	if t.tracerProvidersCounter == 0 {
+		if t.tracerProvider != nil {
+			// tracerProvider.ForceFlush SHOULD be invoked according to https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#forceflush
+			if err := t.tracerProvider.ForceFlush(context.Background()); err != nil {
+				logger.Error("forcing flush", zap.Error(err))
+			}
+
+			// tracerProvider.Shutdown MUST be invoked according to https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#shutdown
+			if err := t.tracerProvider.Shutdown(context.Background()); err != nil {
+				return fmt.Errorf("tracerProvider shutdown error: %w", err)
+			}
+		}
+
+		t.tracerProvider = nil
+	}
+
+	return nil
+}
diff --git a/modules/caddyhttp/tracing/tracerprovider_test.go b/modules/caddyhttp/tracing/tracerprovider_test.go
new file mode 100644
index 000000000..cb2e5936f
--- /dev/null
+++ b/modules/caddyhttp/tracing/tracerprovider_test.go
@@ -0,0 +1,43 @@
+package tracing
+
+import (
+	"testing"
+
+	"go.uber.org/zap"
+)
+
+func Test_tracersProvider_getTracerProvider(t *testing.T) {
+	tp := tracerProvider{}
+
+	tp.getTracerProvider()
+	tp.getTracerProvider()
+
+	if tp.tracerProvider == nil {
+		t.Errorf("There should be tracer provider")
+	}
+
+	if tp.tracerProvidersCounter != 2 {
+		t.Errorf("Tracer providers counter should equal to 2")
+	}
+}
+
+func Test_tracersProvider_cleanupTracerProvider(t *testing.T) {
+	tp := tracerProvider{}
+
+	tp.getTracerProvider()
+	tp.getTracerProvider()
+
+	err := tp.cleanupTracerProvider(zap.NewNop())
+
+	if err != nil {
+		t.Errorf("There should be no error: %v", err)
+	}
+
+	if tp.tracerProvider == nil {
+		t.Errorf("There should be tracer provider")
+	}
+
+	if tp.tracerProvidersCounter != 1 {
+		t.Errorf("Tracer providers counter should equal to 1")
+	}
+}