fileserver: Support virtual file systems (#4909)

* fileserver: Support virtual file systems (close #3720)

This change replaces the hard-coded use of os.Open() and os.Stat() with
the use of the new (Go 1.16) io/fs APIs, enabling virtual file systems.
It introduces a new module namespace, caddy.fs, for such file systems.

Also improve documentation for the file server. I realized it was one of
the first modules written for Caddy 2, and the docs hadn't really been
updated since!

* Virtualize FS for file matcher; minor tweaks

* Fix tests and rename dirFS -> osFS

(Since we do not use a root directory, it is dynamic.)

1 files changed, 27 insertions, 7 deletions

diff --git a/modules/caddyhttp/fileserver/matcher.go b/modules/caddyhttp/fileserver/matcher.go
index 3ef3e47a2..5cade2575 100644
--- a/modules/caddyhttp/fileserver/matcher.go
+++ b/modules/caddyhttp/fileserver/matcher.go
@@ -15,7 +15,9 @@
 package fileserver
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/fs"
 	"net/http"
 	"os"
 	"path"
@@ -54,6 +56,11 @@ func init() {
 // - `{http.matchers.file.remainder}` Set to the remainder
 // of the path if the path was split by `split_path`.
 type MatchFile struct {
+	// The file system implementation to use. By default, the
+	// local disk file system will be used.
+	FileSystemRaw json.RawMessage `json:"file_system,omitempty" caddy:"namespace=caddy.fs inline_key=backend"`
+	fileSystem    fs.StatFS
+
 	// The root directory, used for creating absolute
 	// file paths, and required when working with
 	// relative paths; if not specified, `{http.vars.root}`
@@ -241,10 +248,23 @@ func celFileMatcherMacroExpander() parser.MacroExpander {
 }
 
 // Provision sets up m's defaults.
-func (m *MatchFile) Provision(_ caddy.Context) error {
+func (m *MatchFile) Provision(ctx caddy.Context) error {
+	// establish the file system to use
+	if len(m.FileSystemRaw) > 0 {
+		mod, err := ctx.LoadModule(m, "FileSystemRaw")
+		if err != nil {
+			return fmt.Errorf("loading file system module: %v", err)
+		}
+		m.fileSystem = mod.(fs.StatFS)
+	}
+	if m.fileSystem == nil {
+		m.fileSystem = osFS{}
+	}
+
 	if m.Root == "" {
 		m.Root = "{http.vars.root}"
 	}
+
 	// if list of files to try was omitted entirely, assume URL path
 	// (use placeholder instead of r.URL.Path; see issue #4146)
 	if m.TryFiles == nil {
@@ -316,7 +336,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 				return
 			}
 			suffix, fullpath, remainder := prepareFilePath(f)
-			if info, exists := strictFileExists(fullpath); exists {
+			if info, exists := m.strictFileExists(fullpath); exists {
 				setPlaceholders(info, suffix, fullpath, remainder)
 				return true
 			}
@@ -330,7 +350,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 		var info os.FileInfo
 		for _, f := range m.TryFiles {
 			suffix, fullpath, splitRemainder := prepareFilePath(f)
-			info, err := os.Stat(fullpath)
+			info, err := m.fileSystem.Stat(fullpath)
 			if err == nil && info.Size() > largestSize {
 				largestSize = info.Size()
 				largestFilename = fullpath
@@ -349,7 +369,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 		var info os.FileInfo
 		for _, f := range m.TryFiles {
 			suffix, fullpath, splitRemainder := prepareFilePath(f)
-			info, err := os.Stat(fullpath)
+			info, err := m.fileSystem.Stat(fullpath)
 			if err == nil && (smallestSize == 0 || info.Size() < smallestSize) {
 				smallestSize = info.Size()
 				smallestFilename = fullpath
@@ -368,7 +388,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
 		var info os.FileInfo
 		for _, f := range m.TryFiles {
 			suffix, fullpath, splitRemainder := prepareFilePath(f)
-			info, err := os.Stat(fullpath)
+			info, err := m.fileSystem.Stat(fullpath)
 			if err == nil &&
 				(recentDate.IsZero() || info.ModTime().After(recentDate)) {
 				recentDate = info.ModTime()
@@ -404,8 +424,8 @@ func parseErrorCode(input string) error {
 // the file must also be a directory; if it does
 // NOT end in a forward slash, the file must NOT
 // be a directory.
-func strictFileExists(file string) (os.FileInfo, bool) {
-	stat, err := os.Stat(file)
+func (m MatchFile) strictFileExists(file string) (os.FileInfo, bool) {
+	stat, err := m.fileSystem.Stat(file)
 	if err != nil {
 		// in reality, this can be any error
 		// such as permission or even obscure