caddyhttp: Fix h3 shutdown (#5541)

* swap h3server close and listener close, avoid quic-listener not closing

* fix typo

2 files changed, 16 insertions, 8 deletions

diff --git a/modules/caddyhttp/app.go b/modules/caddyhttp/app.go
index 0e02afdd3..f774c3ae4 100644
--- a/modules/caddyhttp/app.go
+++ b/modules/caddyhttp/app.go
@@ -594,6 +594,21 @@ func (app *App) Stop() error {
 			return
 		}
 
+		// First close h3server then close listeners unlike stdlib for several reasons:
+		// 1, udp has only a single socket, once closed, no more data can be read and
+		// written. In contrast, closing tcp listeners won't affect established connections.
+		// This have something to do with graceful shutdown when upstream implements it.
+		// 2, h3server will only close listeners it's registered (quic listeners). Closing
+		// listener first and these listeners maybe unregistered thus won't be closed. caddy
+		// distinguishes quic-listener and underlying datagram sockets.
+
+		// TODO: CloseGracefully, once implemented upstream (see https://github.com/quic-go/quic-go/issues/2103)
+		if err := server.h3server.Close(); err != nil {
+			app.logger.Error("HTTP/3 server shutdown",
+				zap.Error(err),
+				zap.Strings("addresses", server.Listen))
+		}
+
 		// TODO: we have to manually close our listeners because quic-go won't
 		// close listeners it didn't create along with the server itself...
 		// see https://github.com/quic-go/quic-go/issues/3560
@@ -604,13 +619,6 @@ func (app *App) Stop() error {
 					zap.String("address", el.LocalAddr().String()))
 			}
 		}
-
-		// TODO: CloseGracefully, once implemented upstream (see https://github.com/quic-go/quic-go/issues/2103)
-		if err := server.h3server.Close(); err != nil {
-			app.logger.Error("HTTP/3 server shutdown",
-				zap.Error(err),
-				zap.Strings("addresses", server.Listen))
-		}
 	}
 	stopH2Listener := func(server *Server) {
 		defer finishedShutdown.Done()
diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go
index d2de09b55..db370db67 100644
--- a/modules/caddyhttp/server.go
+++ b/modules/caddyhttp/server.go
@@ -555,7 +555,7 @@ func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error
 		}
 	}
 
-	s.h3listeners = append(s.h3listeners, lnAny.(net.PacketConn))
+	s.h3listeners = append(s.h3listeners, ln)
 
 	//nolint:errcheck
 	go s.h3server.ServeListener(h3ln)