metrics: scope metrics to active config, add optional per-host metrics (#6531)v2.9.0-beta.1

* Add per host config

* Pass host label when option is enabled

* Test per host enabled

* metrics: scope metrics per loaded config

* doc and linter

Signed-off-by: Mohammed Al Sahaf <[email protected]>

* inject the custom registry into the admin handler

Co-Authored-By: Dave Henderson <[email protected]>

* remove `TODO` comment

* fixes

Signed-off-by: Mohammed Al Sahaf <[email protected]>

* refactor to delay metrics admin handler provision

Signed-off-by: Mohammed Al Sahaf <[email protected]>

---------

Signed-off-by: Mohammed Al Sahaf <[email protected]>
Co-authored-by: Hussam Almarzooq <[email protected]>
Co-authored-by: Dave Henderson <[email protected]>

6 files changed, 272 insertions, 45 deletions

diff --git a/modules/caddyhttp/app.go b/modules/caddyhttp/app.go
index 7a5c10623..7dc2bee72 100644
--- a/modules/caddyhttp/app.go
+++ b/modules/caddyhttp/app.go
@@ -347,6 +347,10 @@ func (app *App) Provision(ctx caddy.Context) error {
 		// route handler so that important security checks are done, etc.
 		primaryRoute := emptyHandler
 		if srv.Routes != nil {
+			if srv.Metrics != nil {
+				srv.Metrics.init = sync.Once{}
+				srv.Metrics.httpMetrics = &httpMetrics{}
+			}
 			err := srv.Routes.ProvisionHandlers(ctx, srv.Metrics)
 			if err != nil {
 				return fmt.Errorf("server %s: setting up route handlers: %v", srvName, err)
diff --git a/modules/caddyhttp/metrics.go b/modules/caddyhttp/metrics.go
index 111389218..947721429 100644
--- a/modules/caddyhttp/metrics.go
+++ b/modules/caddyhttp/metrics.go
@@ -10,15 +10,23 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
+	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/internal/metrics"
 )
 
 // Metrics configures metrics observations.
 // EXPERIMENTAL and subject to change or removal.
-type Metrics struct{}
+type Metrics struct {
+	// Enable per-host metrics. Enabling this option may
+	// incur high-memory consumption, depending on the number of hosts
+	// managed by Caddy.
+	PerHost bool `json:"per_host,omitempty"`
+
+	init        sync.Once
+	httpMetrics *httpMetrics `json:"-"`
+}
 
-var httpMetrics = struct {
-	init             sync.Once
+type httpMetrics struct {
 	requestInFlight  *prometheus.GaugeVec
 	requestCount     *prometheus.CounterVec
 	requestErrors    *prometheus.CounterVec
@@ -26,27 +34,28 @@ var httpMetrics = struct {
 	requestSize      *prometheus.HistogramVec
 	responseSize     *prometheus.HistogramVec
 	responseDuration *prometheus.HistogramVec
-}{
-	init: sync.Once{},
 }
 
-func initHTTPMetrics() {
+func initHTTPMetrics(ctx caddy.Context, metrics *Metrics) {
 	const ns, sub = "caddy", "http"
-
+	registry := ctx.GetMetricsRegistry()
 	basicLabels := []string{"server", "handler"}
-	httpMetrics.requestInFlight = promauto.NewGaugeVec(prometheus.GaugeOpts{
+	if metrics.PerHost {
+		basicLabels = append(basicLabels, "host")
+	}
+	metrics.httpMetrics.requestInFlight = promauto.With(registry).NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: ns,
 		Subsystem: sub,
 		Name:      "requests_in_flight",
 		Help:      "Number of requests currently handled by this server.",
 	}, basicLabels)
-	httpMetrics.requestErrors = promauto.NewCounterVec(prometheus.CounterOpts{
+	metrics.httpMetrics.requestErrors = promauto.With(registry).NewCounterVec(prometheus.CounterOpts{
 		Namespace: ns,
 		Subsystem: sub,
 		Name:      "request_errors_total",
 		Help:      "Number of requests resulting in middleware errors.",
 	}, basicLabels)
-	httpMetrics.requestCount = promauto.NewCounterVec(prometheus.CounterOpts{
+	metrics.httpMetrics.requestCount = promauto.With(registry).NewCounterVec(prometheus.CounterOpts{
 		Namespace: ns,
 		Subsystem: sub,
 		Name:      "requests_total",
@@ -58,28 +67,31 @@ func initHTTPMetrics() {
 	sizeBuckets := prometheus.ExponentialBuckets(256, 4, 8)
 
 	httpLabels := []string{"server", "handler", "code", "method"}
-	httpMetrics.requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	if metrics.PerHost {
+		httpLabels = append(httpLabels, "host")
+	}
+	metrics.httpMetrics.requestDuration = promauto.With(registry).NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: ns,
 		Subsystem: sub,
 		Name:      "request_duration_seconds",
 		Help:      "Histogram of round-trip request durations.",
 		Buckets:   durationBuckets,
 	}, httpLabels)
-	httpMetrics.requestSize = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	metrics.httpMetrics.requestSize = promauto.With(registry).NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: ns,
 		Subsystem: sub,
 		Name:      "request_size_bytes",
 		Help:      "Total size of the request. Includes body",
 		Buckets:   sizeBuckets,
 	}, httpLabels)
-	httpMetrics.responseSize = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	metrics.httpMetrics.responseSize = promauto.With(registry).NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: ns,
 		Subsystem: sub,
 		Name:      "response_size_bytes",
 		Help:      "Size of the returned response.",
 		Buckets:   sizeBuckets,
 	}, httpLabels)
-	httpMetrics.responseDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	metrics.httpMetrics.responseDuration = promauto.With(registry).NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: ns,
 		Subsystem: sub,
 		Name:      "response_duration_seconds",
@@ -101,14 +113,15 @@ func serverNameFromContext(ctx context.Context) string {
 type metricsInstrumentedHandler struct {
 	handler string
 	mh      MiddlewareHandler
+	metrics *Metrics
 }
 
-func newMetricsInstrumentedHandler(handler string, mh MiddlewareHandler) *metricsInstrumentedHandler {
-	httpMetrics.init.Do(func() {
-		initHTTPMetrics()
+func newMetricsInstrumentedHandler(ctx caddy.Context, handler string, mh MiddlewareHandler, metrics *Metrics) *metricsInstrumentedHandler {
+	metrics.init.Do(func() {
+		initHTTPMetrics(ctx, metrics)
 	})
 
-	return &metricsInstrumentedHandler{handler, mh}
+	return &metricsInstrumentedHandler{handler, mh, metrics}
 }
 
 func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error {
@@ -119,7 +132,12 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
 	// of a panic
 	statusLabels := prometheus.Labels{"server": server, "handler": h.handler, "method": method, "code": ""}
 
-	inFlight := httpMetrics.requestInFlight.With(labels)
+	if h.metrics.PerHost {
+		labels["host"] = r.Host
+		statusLabels["host"] = r.Host
+	}
+
+	inFlight := h.metrics.httpMetrics.requestInFlight.With(labels)
 	inFlight.Inc()
 	defer inFlight.Dec()
 
@@ -131,13 +149,13 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
 	writeHeaderRecorder := ShouldBufferFunc(func(status int, header http.Header) bool {
 		statusLabels["code"] = metrics.SanitizeCode(status)
 		ttfb := time.Since(start).Seconds()
-		httpMetrics.responseDuration.With(statusLabels).Observe(ttfb)
+		h.metrics.httpMetrics.responseDuration.With(statusLabels).Observe(ttfb)
 		return false
 	})
 	wrec := NewResponseRecorder(w, nil, writeHeaderRecorder)
 	err := h.mh.ServeHTTP(wrec, r, next)
 	dur := time.Since(start).Seconds()
-	httpMetrics.requestCount.With(labels).Inc()
+	h.metrics.httpMetrics.requestCount.With(labels).Inc()
 
 	observeRequest := func(status int) {
 		// If the code hasn't been set yet, and we didn't encounter an error, we're
@@ -148,9 +166,9 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
 			statusLabels["code"] = metrics.SanitizeCode(status)
 		}
 
-		httpMetrics.requestDuration.With(statusLabels).Observe(dur)
-		httpMetrics.requestSize.With(statusLabels).Observe(float64(computeApproximateRequestSize(r)))
-		httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size()))
+		h.metrics.httpMetrics.requestDuration.With(statusLabels).Observe(dur)
+		h.metrics.httpMetrics.requestSize.With(statusLabels).Observe(float64(computeApproximateRequestSize(r)))
+		h.metrics.httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size()))
 	}
 
 	if err != nil {
@@ -159,7 +177,7 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
 			observeRequest(handlerErr.StatusCode)
 		}
 
-		httpMetrics.requestErrors.With(labels).Inc()
+		h.metrics.httpMetrics.requestErrors.With(labels).Inc()
 
 		return err
 	}
diff --git a/modules/caddyhttp/metrics_test.go b/modules/caddyhttp/metrics_test.go
index 8f88549d5..4a0519b87 100644
--- a/modules/caddyhttp/metrics_test.go
+++ b/modules/caddyhttp/metrics_test.go
@@ -6,9 +6,10 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync"
 	"testing"
 
-	"github.com/prometheus/client_golang/prometheus"
+	"github.com/caddyserver/caddy/v2"
 	"github.com/prometheus/client_golang/prometheus/testutil"
 )
 
@@ -27,10 +28,15 @@ func TestServerNameFromContext(t *testing.T) {
 }
 
 func TestMetricsInstrumentedHandler(t *testing.T) {
+	ctx, _ := caddy.NewContext(caddy.Context{Context: context.Background()})
+	metrics := &Metrics{
+		init:        sync.Once{},
+		httpMetrics: &httpMetrics{},
+	}
 	handlerErr := errors.New("oh noes")
 	response := []byte("hello world!")
 	h := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-		if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 1.0 {
+		if actual := testutil.ToFloat64(metrics.httpMetrics.requestInFlight); actual != 1.0 {
 			t.Errorf("Not same: expected %#v, but got %#v", 1.0, actual)
 		}
 		if handlerErr == nil {
@@ -43,7 +49,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
 		return h.ServeHTTP(w, r)
 	})
 
-	ih := newMetricsInstrumentedHandler("bar", mh)
+	ih := newMetricsInstrumentedHandler(ctx, "bar", mh, metrics)
 
 	r := httptest.NewRequest("GET", "/", nil)
 	w := httptest.NewRecorder()
@@ -51,7 +57,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
 	if actual := ih.ServeHTTP(w, r, h); actual != handlerErr {
 		t.Errorf("Not same: expected %#v, but got %#v", handlerErr, actual)
 	}
-	if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 0.0 {
+	if actual := testutil.ToFloat64(metrics.httpMetrics.requestInFlight); actual != 0.0 {
 		t.Errorf("Not same: expected %#v, but got %#v", 0.0, actual)
 	}
 
@@ -64,7 +70,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
 	mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
 		return nil
 	})
-	ih = newMetricsInstrumentedHandler("empty", mh)
+	ih = newMetricsInstrumentedHandler(ctx, "empty", mh, metrics)
 	r = httptest.NewRequest("GET", "/", nil)
 	w = httptest.NewRecorder()
 
@@ -83,7 +89,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
 		return Error(http.StatusTooManyRequests, nil)
 	})
 
-	ih = newMetricsInstrumentedHandler("foo", mh)
+	ih = newMetricsInstrumentedHandler(ctx, "foo", mh, metrics)
 
 	r = httptest.NewRequest("GET", "/", nil)
 	w = httptest.NewRecorder()
@@ -183,7 +189,183 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
 	caddy_http_request_errors_total{handler="bar",server="UNKNOWN"} 1
 	caddy_http_request_errors_total{handler="foo",server="UNKNOWN"} 1
 	`
-	if err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected),
+	if err := testutil.GatherAndCompare(ctx.GetMetricsRegistry(), strings.NewReader(expected),
+		"caddy_http_request_size_bytes",
+		"caddy_http_response_size_bytes",
+		// caddy_http_request_duration_seconds_sum will vary based on how long the test took to run,
+		// so we check just the _bucket and _count metrics
+		"caddy_http_request_duration_seconds_bucket",
+		"caddy_http_request_duration_seconds_count",
+		"caddy_http_request_errors_total",
+	); err != nil {
+		t.Errorf("received unexpected error: %s", err)
+	}
+}
+
+func TestMetricsInstrumentedHandlerPerHost(t *testing.T) {
+	ctx, _ := caddy.NewContext(caddy.Context{Context: context.Background()})
+	metrics := &Metrics{
+		PerHost:     true,
+		init:        sync.Once{},
+		httpMetrics: &httpMetrics{},
+	}
+	handlerErr := errors.New("oh noes")
+	response := []byte("hello world!")
+	h := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+		if actual := testutil.ToFloat64(metrics.httpMetrics.requestInFlight); actual != 1.0 {
+			t.Errorf("Not same: expected %#v, but got %#v", 1.0, actual)
+		}
+		if handlerErr == nil {
+			w.Write(response)
+		}
+		return handlerErr
+	})
+
+	mh := middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
+		return h.ServeHTTP(w, r)
+	})
+
+	ih := newMetricsInstrumentedHandler(ctx, "bar", mh, metrics)
+
+	r := httptest.NewRequest("GET", "/", nil)
+	w := httptest.NewRecorder()
+
+	if actual := ih.ServeHTTP(w, r, h); actual != handlerErr {
+		t.Errorf("Not same: expected %#v, but got %#v", handlerErr, actual)
+	}
+	if actual := testutil.ToFloat64(metrics.httpMetrics.requestInFlight); actual != 0.0 {
+		t.Errorf("Not same: expected %#v, but got %#v", 0.0, actual)
+	}
+
+	handlerErr = nil
+	if err := ih.ServeHTTP(w, r, h); err != nil {
+		t.Errorf("Received unexpected error: %v", err)
+	}
+
+	// an empty handler - no errors, no header written
+	mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
+		return nil
+	})
+	ih = newMetricsInstrumentedHandler(ctx, "empty", mh, metrics)
+	r = httptest.NewRequest("GET", "/", nil)
+	w = httptest.NewRecorder()
+
+	if err := ih.ServeHTTP(w, r, h); err != nil {
+		t.Errorf("Received unexpected error: %v", err)
+	}
+	if actual := w.Result().StatusCode; actual != 200 {
+		t.Errorf("Not same: expected status code %#v, but got %#v", 200, actual)
+	}
+	if actual := w.Result().Header; len(actual) != 0 {
+		t.Errorf("Not empty: expected headers to be empty, but got %#v", actual)
+	}
+
+	// handler returning an error with an HTTP status
+	mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
+		return Error(http.StatusTooManyRequests, nil)
+	})
+
+	ih = newMetricsInstrumentedHandler(ctx, "foo", mh, metrics)
+
+	r = httptest.NewRequest("GET", "/", nil)
+	w = httptest.NewRecorder()
+
+	if err := ih.ServeHTTP(w, r, nil); err == nil {
+		t.Errorf("expected error to be propagated")
+	}
+
+	expected := `
+	# HELP caddy_http_request_duration_seconds Histogram of round-trip request durations.
+	# TYPE caddy_http_request_duration_seconds histogram
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.005"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.01"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.025"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.05"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.1"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.25"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.5"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="2.5"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="5"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="10"} 1
+	caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+	caddy_http_request_duration_seconds_count{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 1
+	# HELP caddy_http_request_size_bytes Total size of the request. Includes body
+	# TYPE caddy_http_request_size_bytes histogram
+	caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+    caddy_http_request_size_bytes_sum{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 23
+    caddy_http_request_size_bytes_count{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
+    caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+    caddy_http_request_size_bytes_sum{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 23
+    caddy_http_request_size_bytes_count{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
+	caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+	caddy_http_request_size_bytes_sum{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 23
+	caddy_http_request_size_bytes_count{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 1
+	# HELP caddy_http_response_size_bytes Size of the returned response.
+	# TYPE caddy_http_response_size_bytes histogram
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+	caddy_http_response_size_bytes_sum{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 12
+	caddy_http_response_size_bytes_count{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
+	caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+	caddy_http_response_size_bytes_sum{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 0
+	caddy_http_response_size_bytes_count{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
+	caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
+	caddy_http_response_size_bytes_sum{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 0
+	caddy_http_response_size_bytes_count{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 1
+	# HELP caddy_http_request_errors_total Number of requests resulting in middleware errors.
+	# TYPE caddy_http_request_errors_total counter
+	caddy_http_request_errors_total{handler="bar",host="example.com",server="UNKNOWN"} 1
+	caddy_http_request_errors_total{handler="foo",host="example.com",server="UNKNOWN"} 1
+	`
+	if err := testutil.GatherAndCompare(ctx.GetMetricsRegistry(), strings.NewReader(expected),
 		"caddy_http_request_size_bytes",
 		"caddy_http_response_size_bytes",
 		// caddy_http_request_duration_seconds_sum will vary based on how long the test took to run,
diff --git a/modules/caddyhttp/routes.go b/modules/caddyhttp/routes.go
index 54a3f38e6..939d01e55 100644
--- a/modules/caddyhttp/routes.go
+++ b/modules/caddyhttp/routes.go
@@ -314,11 +314,11 @@ func wrapRoute(route Route) Middleware {
 // we need to pull this particular MiddlewareHandler
 // pointer into its own stack frame to preserve it so it
 // won't be overwritten in future loop iterations.
-func wrapMiddleware(_ caddy.Context, mh MiddlewareHandler, metrics *Metrics) Middleware {
+func wrapMiddleware(ctx caddy.Context, mh MiddlewareHandler, metrics *Metrics) Middleware {
 	handlerToUse := mh
 	if metrics != nil {
 		// wrap the middleware with metrics instrumentation
-		handlerToUse = newMetricsInstrumentedHandler(caddy.GetModuleName(mh), mh)
+		handlerToUse = newMetricsInstrumentedHandler(ctx, caddy.GetModuleName(mh), mh, metrics)
 	}
 
 	return func(next Handler) Handler {
diff --git a/modules/metrics/adminmetrics.go b/modules/metrics/adminmetrics.go
index 1cf398e4c..1e3a841dd 100644
--- a/modules/metrics/adminmetrics.go
+++ b/modules/metrics/adminmetrics.go
@@ -15,8 +15,11 @@
 package metrics
 
 import (
+	"errors"
 	"net/http"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/caddyserver/caddy/v2"
 )
 
@@ -29,7 +32,11 @@ func init() {
 // is permanently mounted to the admin API endpoint at "/metrics".
 // See the Metrics module for a configurable endpoint that is usable if the
 // Admin API is disabled.
-type AdminMetrics struct{}
+type AdminMetrics struct {
+	registry *prometheus.Registry
+
+	metricsHandler http.Handler
+}
 
 // CaddyModule returns the Caddy module information.
 func (AdminMetrics) CaddyModule() caddy.ModuleInfo {
@@ -39,17 +46,28 @@ func (AdminMetrics) CaddyModule() caddy.ModuleInfo {
 	}
 }
 
+// Provision -
+func (m *AdminMetrics) Provision(ctx caddy.Context) error {
+	m.registry = ctx.GetMetricsRegistry()
+	if m.registry == nil {
+		return errors.New("no metrics registry found")
+	}
+	m.metricsHandler = createMetricsHandler(nil, false, m.registry)
+	return nil
+}
+
 // Routes returns a route for the /metrics endpoint.
 func (m *AdminMetrics) Routes() []caddy.AdminRoute {
-	metricsHandler := createMetricsHandler(nil, false)
-	h := caddy.AdminHandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-		metricsHandler.ServeHTTP(w, r)
-		return nil
-	})
-	return []caddy.AdminRoute{{Pattern: "/metrics", Handler: h}}
+	return []caddy.AdminRoute{{Pattern: "/metrics", Handler: caddy.AdminHandlerFunc(m.serveHTTP)}}
+}
+
+func (m *AdminMetrics) serveHTTP(w http.ResponseWriter, r *http.Request) error {
+	m.metricsHandler.ServeHTTP(w, r)
+	return nil
 }
 
 // Interface guards
 var (
+	_ caddy.Provisioner = (*AdminMetrics)(nil)
 	_ caddy.AdminRouter = (*AdminMetrics)(nil)
 )
diff --git a/modules/metrics/metrics.go b/modules/metrics/metrics.go
index dc6196a15..42b30d88d 100644
--- a/modules/metrics/metrics.go
+++ b/modules/metrics/metrics.go
@@ -15,6 +15,7 @@
 package metrics
 
 import (
+	"errors"
 	"net/http"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -62,7 +63,11 @@ func (l *zapLogger) Println(v ...any) {
 // Provision sets up m.
 func (m *Metrics) Provision(ctx caddy.Context) error {
 	log := ctx.Logger()
-	m.metricsHandler = createMetricsHandler(&zapLogger{log}, !m.DisableOpenMetrics)
+	registry := ctx.GetMetricsRegistry()
+	if registry == nil {
+		return errors.New("no metrics registry found")
+	}
+	m.metricsHandler = createMetricsHandler(&zapLogger{log}, !m.DisableOpenMetrics, registry)
 	return nil
 }
 
@@ -107,9 +112,9 @@ var (
 	_ caddyfile.Unmarshaler       = (*Metrics)(nil)
 )
 
-func createMetricsHandler(logger promhttp.Logger, enableOpenMetrics bool) http.Handler {
-	return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer,
-		promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{
+func createMetricsHandler(logger promhttp.Logger, enableOpenMetrics bool, registry *prometheus.Registry) http.Handler {
+	return promhttp.InstrumentMetricHandler(registry,
+		promhttp.HandlerFor(registry, promhttp.HandlerOpts{
 			// will only log errors if logger is non-nil
 			ErrorLog: logger,