metrics: scope metrics to active config, add optional per-host metrics (#6531)v2.9.0-beta.1

* Add per host config

* Pass host label when option is enabled

* Test per host enabled

* metrics: scope metrics per loaded config

* doc and linter

Signed-off-by: Mohammed Al Sahaf <[email protected]>

* inject the custom registry into the admin handler

Co-Authored-By: Dave Henderson <[email protected]>

* remove `TODO` comment

* fixes

Signed-off-by: Mohammed Al Sahaf <[email protected]>

* refactor to delay metrics admin handler provision

Signed-off-by: Mohammed Al Sahaf <[email protected]>

---------

Signed-off-by: Mohammed Al Sahaf <[email protected]>
Co-authored-by: Hussam Almarzooq <[email protected]>
Co-authored-by: Dave Henderson <[email protected]>

1 files changed, 189 insertions, 7 deletions

diff --git a/modules/caddyhttp/metrics_test.go b/modules/caddyhttp/metrics_test.go
index 8f88549d5..4a0519b87 100644
--- a/modules/caddyhttp/metrics_test.go
+++ b/modules/caddyhttp/metrics_test.go
@@ -6,9 +6,10 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync"
 	"testing"
 
-	"github.com/prometheus/client_golang/prometheus"
+	"github.com/caddyserver/caddy/v2"
 	"github.com/prometheus/client_golang/prometheus/testutil"
 )
 
@@ -27,10 +28,15 @@ func TestServerNameFromContext(t *testing.T) {
 }
 
 func TestMetricsInstrumentedHandler(t *testing.T) {
+	ctx, _ := caddy.NewContext(caddy.Context{Context: context.Background()})
+	metrics := &Metrics{
+		init:        sync.Once{},
+		httpMetrics: &httpMetrics{},
+	}
 	handlerErr := errors.New("oh noes")
 	response := []byte("hello world!")
 	h := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-		if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 1.0 {
+		if actual := testutil.ToFloat64(metrics.httpMetrics.requestInFlight); actual != 1.0 {
 			t.Errorf("Not same: expected %#v, but got %#v", 1.0, actual)
 		}
 		if handlerErr == nil {
@@ -43,7 +49,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
 		return h.ServeHTTP(w, r)
 	})
 
-	ih := newMetricsInstrumentedHandler("bar", mh)
+	ih := newMetricsInstrumentedHandler(ctx, "bar", mh, metrics)
 
 	r := httptest.NewRequest("GET", "/", nil)
 	w := httptest.NewRecorder()
@@ -51,7 +57,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
 	if actual := ih.ServeHTTP(w, r, h); actual != handlerErr {
 		t.Errorf("Not same: expected %#v, but got %#v", handlerErr, actual)
 	}
-	if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 0.0 {
+	if actual := testutil.ToFloat64(metrics.httpMetrics.requestInFlight); actual != 0.0 {
 		t.Errorf("Not same: expected %#v, but got %#v", 0.0, actual)
 	}
 
@@ -64,7 +70,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
 	mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
 		return nil
 	})
-	ih = newMetricsInstrumentedHandler("empty", mh)
+	ih = newMetricsInstrumentedHandler(ctx, "empty", mh, metrics)
 	r = httptest.NewRequest("GET", "/", nil)
 	w = httptest.NewRecorder()
 
@@ -83,7 +89,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
 		return Error(http.StatusTooManyRequests, nil)
 	})
 
-	ih = newMetricsInstrumentedHandler("foo", mh)
+	ih = newMetricsInstrumentedHandler(ctx, "foo", mh, metrics)
 
 	r = httptest.NewRequest("GET", "/", nil)
 	w = httptest.NewRecorder()
@@ -183,7 +189,183 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
 	caddy_http_request_errors_total{handler="bar",server="UNKNOWN"} 1
 	caddy_http_request_errors_total{handler="foo",server="UNKNOWN"} 1
 	`
-	if err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected),
+	if err := testutil.GatherAndCompare(ctx.GetMetricsRegistry(), strings.NewReader(expected),
+		"caddy_http_request_size_bytes",
+		"caddy_http_response_size_bytes",
+		// caddy_http_request_duration_seconds_sum will vary based on how long the test took to run,
+		// so we check just the _bucket and _count metrics
+		"caddy_http_request_duration_seconds_bucket",
+		"caddy_http_request_duration_seconds_count",
+		"caddy_http_request_errors_total",
+	); err != nil {
+		t.Errorf("received unexpected error: %s", err)
+	}
+}
+
+func TestMetricsInstrumentedHandlerPerHost(t *testing.T) {
+	ctx, _ := caddy.NewContext(caddy.Context{Context: context.Background()})
+	metrics := &Metrics{
+		PerHost:     true,
+		init:        sync.Once{},
+		httpMetrics: &httpMetrics{},
+	}
+	handlerErr := errors.New("oh noes")
+	response := []byte("hello world!")
+	h := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+		if actual := testutil.ToFloat64(metrics.httpMetrics.requestInFlight); actual != 1.0 {
+			t.Errorf("Not same: expected %#v, but got %#v", 1.0, actual)
+		}
+		if handlerErr == nil {
+			w.Write(response)
+		}
+		return handlerErr
+	})
+
+	mh := middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
+		return h.ServeHTTP(w, r)
+	})
+
+	ih := newMetricsInstrumentedHandler(ctx, "bar", mh, metrics)
+
+	r := httptest.NewRequest("GET", "/", nil)
+	w := httptest.NewRecorder()
+
+	if actual := ih.ServeHTTP(w, r, h); actual != handlerErr {
+		t.Errorf("Not same: expected %#v, but got %#v", handlerErr, actual)
+	}
+	if actual := testutil.ToFloat64(metrics.httpMetrics.requestInFlight); actual != 0.0 {
+		t.Errorf("Not same: expected %#v, but got %#v", 0.0, actual)
+	}
+
+	handlerErr = nil
+	if err := ih.ServeHTTP(w, r, h); err != nil {
+		t.Errorf("Received unexpected error: %v", err)
+	}
+
+	// an empty handler - no errors, no header written
+	mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
+		return nil
+	})
+	ih = newMetricsInstrumentedHandler(ctx, "empty", mh, metrics)
+	r = httptest.NewRequest("GET", "/", nil)
+	w = httptest.NewRecorder()
+
+	if err := ih.ServeHTTP(w, r, h); err != nil {
+		t.Errorf("Received unexpected error: %v", err)
+	}
+	if actual := w.Result().StatusCode; actual != 200 {
+		t.Errorf("Not same: expected status code %#v, but got %#v", 200, actual)
+	}
+	if actual := w.Result().Header; len(actual) != 0 {
+		t.Errorf("Not empty: expected headers to be empty, but got %#v", actual)
+	}
+
+	// handler returning an error with an HTTP status
+	mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
+		return Error(http.StatusTooManyRequests, nil)
+	})
+
+	ih = newMetricsInstrumentedHandler(ctx, "foo", mh, metrics)
+
+	r = httptest.NewRequest("GET", "/", nil)
+	w = httptest.NewRecorder()
+
+	if err := ih.ServeHTTP(w, r, nil); err == nil {
+		t.Errorf("expected error to be propagated")
+	}
+
+	expected := `
+	# HELP caddy_http_request_duration_seconds Histogram of round-trip request durations.
+	# TYPE caddy_http_request_duration_seconds histogram
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.005"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.01"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.025"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.05"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.1"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.25"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.5"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="2.5"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="5"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="10"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+	caddy_http_request_duration_seconds_count{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 1
+	# HELP caddy_http_request_size_bytes Total size of the request. Includes body
+	# TYPE caddy_http_request_size_bytes histogram
+	caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+    caddy_http_request_size_bytes_sum{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 23
+    caddy_http_request_size_bytes_count{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+    caddy_http_request_size_bytes_sum{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 23
+    caddy_http_request_size_bytes_count{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+	caddy_http_request_size_bytes_sum{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 23
+	caddy_http_request_size_bytes_count{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 1
+	# HELP caddy_http_response_size_bytes Size of the returned response.
+	# TYPE caddy_http_response_size_bytes histogram
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+	caddy_http_response_size_bytes_sum{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 12
+	caddy_http_response_size_bytes_count{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+	caddy_http_response_size_bytes_sum{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 0
+	caddy_http_response_size_bytes_count{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+	caddy_http_response_size_bytes_sum{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 0
+	caddy_http_response_size_bytes_count{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 1
+	# HELP caddy_http_request_errors_total Number of requests resulting in middleware errors.
+	# TYPE caddy_http_request_errors_total counter
+	caddy_http_request_errors_total{handler="bar",host="example.com",server="UNKNOWN"} 1
+	caddy_http_request_errors_total{handler="foo",host="example.com",server="UNKNOWN"} 1
+	`
+	if err := testutil.GatherAndCompare(ctx.GetMetricsRegistry(), strings.NewReader(expected),
 		"caddy_http_request_size_bytes",
 		"caddy_http_response_size_bytes",
 		// caddy_http_request_duration_seconds_sum will vary based on how long the test took to run,