Merge branch 'master' into improve-goreleaser-check

13 files changed, 140 insertions, 11 deletions

diff --git a/caddyconfig/httpcaddyfile/options.go b/caddyconfig/httpcaddyfile/options.go
index 03b9ba230..cfd8f709d 100644
--- a/caddyconfig/httpcaddyfile/options.go
+++ b/caddyconfig/httpcaddyfile/options.go
@@ -39,7 +39,8 @@ func init() {
 	RegisterGlobalOption("fallback_sni", parseOptSingleString)
 	RegisterGlobalOption("order", parseOptOrder)
 	RegisterGlobalOption("storage", parseOptStorage)
-	RegisterGlobalOption("storage_clean_interval", parseOptDuration)
+	RegisterGlobalOption("storage_check", parseStorageCheck)
+	RegisterGlobalOption("storage_clean_interval", parseStorageCleanInterval)
 	RegisterGlobalOption("renew_interval", parseOptDuration)
 	RegisterGlobalOption("ocsp_interval", parseOptDuration)
 	RegisterGlobalOption("acme_ca", parseOptSingleString)
@@ -189,6 +190,40 @@ func parseOptStorage(d *caddyfile.Dispenser, _ any) (any, error) {
 	return storage, nil
 }
 
+func parseStorageCheck(d *caddyfile.Dispenser, _ any) (any, error) {
+	d.Next() // consume option name
+	if !d.Next() {
+		return "", d.ArgErr()
+	}
+	val := d.Val()
+	if d.Next() {
+		return "", d.ArgErr()
+	}
+	if val != "off" {
+		return "", d.Errf("storage_check must be 'off'")
+	}
+	return val, nil
+}
+
+func parseStorageCleanInterval(d *caddyfile.Dispenser, _ any) (any, error) {
+	d.Next() // consume option name
+	if !d.Next() {
+		return "", d.ArgErr()
+	}
+	val := d.Val()
+	if d.Next() {
+		return "", d.ArgErr()
+	}
+	if val == "off" {
+		return false, nil
+	}
+	dur, err := caddy.ParseDuration(d.Val())
+	if err != nil {
+		return nil, d.Errf("failed to parse storage_clean_interval, must be a duration or 'off' %w", err)
+	}
+	return caddy.Duration(dur), nil
+}
+
 func parseOptDuration(d *caddyfile.Dispenser, _ any) (any, error) {
 	if !d.Next() { // consume option name
 		return nil, d.ArgErr()
diff --git a/caddyconfig/httpcaddyfile/tlsapp.go b/caddyconfig/httpcaddyfile/tlsapp.go
index bec860610..ea5ac92c7 100644
--- a/caddyconfig/httpcaddyfile/tlsapp.go
+++ b/caddyconfig/httpcaddyfile/tlsapp.go
@@ -349,6 +349,16 @@ func (st ServerType) buildTLSApp(
 		tlsApp.Automation.OnDemand = onDemand
 	}
 
+	// if the storage clean interval is a boolean, then it's "off" to disable cleaning
+	if sc, ok := options["storage_check"].(string); ok && sc == "off" {
+		tlsApp.DisableStorageCheck = true
+	}
+
+	// if the storage clean interval is a boolean, then it's "off" to disable cleaning
+	if sci, ok := options["storage_clean_interval"].(bool); ok && !sci {
+		tlsApp.DisableStorageClean = true
+	}
+
 	// set the storage clean interval if configured
 	if storageCleanInterval, ok := options["storage_clean_interval"].(caddy.Duration); ok {
 		if tlsApp.Automation == nil {
diff --git a/caddytest/integration/caddyfile_adapt/file_server_file_limit.caddyfiletest b/caddytest/integration/caddyfile_adapt/file_server_file_limit.caddyfiletest
new file mode 100644
index 000000000..cd73fbff6
--- /dev/null
+++ b/caddytest/integration/caddyfile_adapt/file_server_file_limit.caddyfiletest
@@ -0,0 +1,36 @@
+:80
+
+file_server {
+	browse {
+		file_limit 4000
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"browse": {
+										"file_limit": 4000
+									},
+									"handler": "file_server",
+									"hide": [
+										"./Caddyfile"
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}
\ No newline at end of file
diff --git a/caddytest/integration/caddyfile_adapt/global_options.caddyfiletest b/caddytest/integration/caddyfile_adapt/global_options.caddyfiletest
index af301615b..99f45cdd5 100644
--- a/caddytest/integration/caddyfile_adapt/global_options.caddyfiletest
+++ b/caddytest/integration/caddyfile_adapt/global_options.caddyfiletest
@@ -9,6 +9,8 @@
 	storage file_system {
 		root /data
 	}
+	storage_check off
+	storage_clean_interval off
 	acme_ca https://example.com
 	acme_ca_root /path/to/ca.crt
 	ocsp_stapling off
@@ -73,7 +75,9 @@
 					}
 				}
 			},
-			"disable_ocsp_stapling": true
+			"disable_ocsp_stapling": true,
+			"disable_storage_check": true,
+			"disable_storage_clean": true
 		}
 	}
 }
diff --git a/go.mod b/go.mod
index d85d22312..7cc0a56b8 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.3.0
 	github.com/alecthomas/chroma/v2 v2.14.0
 	github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b
-	github.com/caddyserver/certmagic v0.21.5-0.20241104195704-c1f1d529a1c2
+	github.com/caddyserver/certmagic v0.21.5-0.20241105180249-4293198e094d
 	github.com/caddyserver/zerossl v0.1.3
 	github.com/dustin/go-humanize v1.0.1
 	github.com/go-chi/chi/v5 v5.0.12
diff --git a/go.sum b/go.sum
index 7f88eb3c2..b264c3424 100644
--- a/go.sum
+++ b/go.sum
@@ -89,8 +89,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
-github.com/caddyserver/certmagic v0.21.5-0.20241104195704-c1f1d529a1c2 h1:g3qaTziJPSGLr+V7zuCWFxQJFDa7/fcmvoQX2rfmBcY=
-github.com/caddyserver/certmagic v0.21.5-0.20241104195704-c1f1d529a1c2/go.mod h1:swUXjQ1T9ZtMv95qj7/InJvWLXURU85r+CfG0T+ZbDE=
+github.com/caddyserver/certmagic v0.21.5-0.20241105180249-4293198e094d h1:+zOduGxxC4WBAnlDf5Uf0TXbWXRqjUXkJKevDZZa79A=
+github.com/caddyserver/certmagic v0.21.5-0.20241105180249-4293198e094d/go.mod h1:swUXjQ1T9ZtMv95qj7/InJvWLXURU85r+CfG0T+ZbDE=
 github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA=
 github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
diff --git a/modules/caddyhttp/fileserver/browse.go b/modules/caddyhttp/fileserver/browse.go
index a19b4e17a..0a623c79e 100644
--- a/modules/caddyhttp/fileserver/browse.go
+++ b/modules/caddyhttp/fileserver/browse.go
@@ -66,8 +66,15 @@ type Browse struct {
 	//   - `sort size` will sort by size in ascending order
 	// The first option must be `sort_by` and the second option must be `order` (if exists).
 	SortOptions []string `json:"sort,omitempty"`
+
+	// FileLimit limits the number of up to n DirEntry values in directory order.
+	FileLimit int `json:"file_limit,omitempty"`
 }
 
+const (
+	defaultDirEntryLimit = 10000
+)
+
 func (fsrv *FileServer) serveBrowse(fileSystem fs.FS, root, dirPath string, w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
 	if c := fsrv.logger.Check(zapcore.DebugLevel, "browse enabled; listing directory contents"); c != nil {
 		c.Write(zap.String("path", dirPath), zap.String("root", root))
@@ -206,7 +213,11 @@ func (fsrv *FileServer) serveBrowse(fileSystem fs.FS, root, dirPath string, w ht
 }
 
 func (fsrv *FileServer) loadDirectoryContents(ctx context.Context, fileSystem fs.FS, dir fs.ReadDirFile, root, urlPath string, repl *caddy.Replacer) (*browseTemplateContext, error) {
-	files, err := dir.ReadDir(10000) // TODO: this limit should probably be configurable
+	dirLimit := defaultDirEntryLimit
+	if fsrv.Browse.FileLimit != 0 {
+		dirLimit = fsrv.Browse.FileLimit
+	}
+	files, err := dir.ReadDir(dirLimit)
 	if err != nil && err != io.EOF {
 		return nil, err
 	}
diff --git a/modules/caddyhttp/fileserver/caddyfile.go b/modules/caddyhttp/fileserver/caddyfile.go
index 81c61611a..dbe2b2e56 100644
--- a/modules/caddyhttp/fileserver/caddyfile.go
+++ b/modules/caddyhttp/fileserver/caddyfile.go
@@ -16,6 +16,7 @@ package fileserver
 
 import (
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
@@ -129,6 +130,16 @@ func (fsrv *FileServer) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 							return d.Errf("unknown sort option '%s'", dVal)
 						}
 					}
+				case "file_limit":
+					fileLimit := d.RemainingArgs()
+					if len(fileLimit) != 1 {
+						return d.Err("file_limit should have an integer value")
+					}
+					val, _ := strconv.Atoi(fileLimit[0])
+					if fsrv.Browse.FileLimit != 0 {
+						return d.Err("file_limit is already enabled")
+					}
+					fsrv.Browse.FileLimit = val
 				default:
 					return d.Errf("unknown subdirective '%s'", d.Val())
 				}
diff --git a/modules/caddyhttp/fileserver/command.go b/modules/caddyhttp/fileserver/command.go
index a76998405..a04d7cade 100644
--- a/modules/caddyhttp/fileserver/command.go
+++ b/modules/caddyhttp/fileserver/command.go
@@ -66,6 +66,7 @@ respond with a file listing.`,
 			cmd.Flags().BoolP("templates", "t", false, "Enable template rendering")
 			cmd.Flags().BoolP("access-log", "a", false, "Enable the access log")
 			cmd.Flags().BoolP("debug", "v", false, "Enable verbose debug logs")
+			cmd.Flags().IntP("file-limit", "f", defaultDirEntryLimit, "Max directories to read")
 			cmd.Flags().BoolP("no-compress", "", false, "Disable Zstandard and Gzip compression")
 			cmd.Flags().StringSliceP("precompressed", "p", []string{}, "Specify precompression file extensions. Compression preference implied from flag order.")
 			cmd.RunE = caddycmd.WrapCommandFuncForCobra(cmdFileServer)
@@ -91,6 +92,7 @@ func cmdFileServer(fs caddycmd.Flags) (int, error) {
 	browse := fs.Bool("browse")
 	templates := fs.Bool("templates")
 	accessLog := fs.Bool("access-log")
+	fileLimit := fs.Int("file-limit")
 	debug := fs.Bool("debug")
 	revealSymlinks := fs.Bool("reveal-symlinks")
 	compress := !fs.Bool("no-compress")
@@ -151,7 +153,7 @@ func cmdFileServer(fs caddycmd.Flags) (int, error) {
 	}
 
 	if browse {
-		handler.Browse = &Browse{RevealSymlinks: revealSymlinks}
+		handler.Browse = &Browse{RevealSymlinks: revealSymlinks, FileLimit: fileLimit}
 	}
 
 	handlers = append(handlers, caddyconfig.JSONModuleObject(handler, "handler", "file_server", nil))
diff --git a/modules/caddyhttp/rewrite/caddyfile.go b/modules/caddyhttp/rewrite/caddyfile.go
index 89f44c79b..5f9b97adf 100644
--- a/modules/caddyhttp/rewrite/caddyfile.go
+++ b/modules/caddyhttp/rewrite/caddyfile.go
@@ -110,9 +110,6 @@ func parseCaddyfileURI(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, err
 			return nil, h.ArgErr()
 		}
 		rewr.StripPathPrefix = args[1]
-		if !strings.HasPrefix(rewr.StripPathPrefix, "/") {
-			rewr.StripPathPrefix = "/" + rewr.StripPathPrefix
-		}
 
 	case "strip_suffix":
 		if len(args) != 2 {
diff --git a/modules/caddyhttp/rewrite/rewrite.go b/modules/caddyhttp/rewrite/rewrite.go
index e76682729..31ebfb430 100644
--- a/modules/caddyhttp/rewrite/rewrite.go
+++ b/modules/caddyhttp/rewrite/rewrite.go
@@ -259,6 +259,9 @@ func (rewr Rewrite) Rewrite(r *http.Request, repl *caddy.Replacer) bool {
 	// strip path prefix or suffix
 	if rewr.StripPathPrefix != "" {
 		prefix := repl.ReplaceAll(rewr.StripPathPrefix, "")
+		if !strings.HasPrefix(prefix, "/") {
+			prefix = "/" + prefix
+		}
 		mergeSlashes := !strings.Contains(prefix, "//")
 		changePath(r, func(escapedPath string) string {
 			escapedPath = caddyhttp.CleanPath(escapedPath, mergeSlashes)
diff --git a/modules/caddyhttp/rewrite/rewrite_test.go b/modules/caddyhttp/rewrite/rewrite_test.go
index aaa142bc2..81360baee 100644
--- a/modules/caddyhttp/rewrite/rewrite_test.go
+++ b/modules/caddyhttp/rewrite/rewrite_test.go
@@ -236,6 +236,11 @@ func TestRewrite(t *testing.T) {
 			expect: newRequest(t, "GET", "/foo/bar"),
 		},
 		{
+			rule:   Rewrite{StripPathPrefix: "prefix"},
+			input:  newRequest(t, "GET", "/prefix/foo/bar"),
+			expect: newRequest(t, "GET", "/foo/bar"),
+		},
+		{
 			rule:   Rewrite{StripPathPrefix: "/prefix"},
 			input:  newRequest(t, "GET", "/prefix"),
 			expect: newRequest(t, "GET", ""),
diff --git a/modules/caddytls/tls.go b/modules/caddytls/tls.go
index 6e660dea8..abb519eb7 100644
--- a/modules/caddytls/tls.go
+++ b/modules/caddytls/tls.go
@@ -92,6 +92,17 @@ type TLS struct {
 	// EXPERIMENTAL. Subject to change.
 	DisableStorageCheck bool `json:"disable_storage_check,omitempty"`
 
+	// Disables the automatic cleanup of the storage backend.
+	// This is useful when TLS is not being used to store certificates
+	// and the user wants run their server in a read-only mode.
+	//
+	// Storage cleaning creates two files: instance.uuid and last_clean.json.
+	// The instance.uuid file is used to identify the instance of Caddy
+	// in a cluster. The last_clean.json file is used to store the last
+	// time the storage was cleaned.
+	// EXPERIMENTAL. Subject to change.
+	DisableStorageClean bool `json:"disable_storage_clean,omitempty"`
+
 	certificateLoaders []CertificateLoader
 	automateNames      []string
 	ctx                caddy.Context
@@ -328,7 +339,11 @@ func (t *TLS) Start() error {
 		return fmt.Errorf("automate: managing %v: %v", t.automateNames, err)
 	}
 
-	t.keepStorageClean()
+	if !t.DisableStorageClean {
+		// start the storage cleaner goroutine and ticker,
+		// which cleans out expired certificates and more
+		t.keepStorageClean()
+	}
 
 	return nil
 }