Add Hugo Modules

This commit implements Hugo Modules.

This is a broad subject, but some keywords include:

* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`,  `hugo mod get`,  `hugo mod graph`,  `hugo mod tidy`, and  `hugo mod vendor`.

All of the above is backed by Go Modules.

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092

6 files changed, 223 insertions, 119 deletions

diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go
index 6ad417117..bf004c8f7 100644
--- a/cache/filecache/filecache.go
+++ b/cache/filecache/filecache.go
@@ -44,6 +44,9 @@ type Cache struct {
 	// 0 is effectively turning this cache off.
 	maxAge time.Duration
 
+	// When set, we just remove this entire root directory on expiration.
+	pruneAllRootDir string
+
 	nlocker *lockTracker
 }
 
@@ -77,11 +80,12 @@ type ItemInfo struct {
 }
 
 // NewCache creates a new file cache with the given filesystem and max age.
-func NewCache(fs afero.Fs, maxAge time.Duration) *Cache {
+func NewCache(fs afero.Fs, maxAge time.Duration, pruneAllRootDir string) *Cache {
 	return &Cache{
-		Fs:      fs,
-		nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
-		maxAge:  maxAge,
+		Fs:              fs,
+		nlocker:         &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
+		maxAge:          maxAge,
+		pruneAllRootDir: pruneAllRootDir,
 	}
 }
 
@@ -307,9 +311,15 @@ func (f Caches) Get(name string) *Cache {
 // NewCaches creates a new set of file caches from the given
 // configuration.
 func NewCaches(p *helpers.PathSpec) (Caches, error) {
-	dcfg, err := decodeConfig(p)
-	if err != nil {
-		return nil, err
+	var dcfg Configs
+	if c, ok := p.Cfg.Get("filecacheConfigs").(Configs); ok {
+		dcfg = c
+	} else {
+		var err error
+		dcfg, err = DecodeConfig(p.Fs.Source, p.Cfg)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	fs := p.Fs.Source
@@ -319,30 +329,25 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
 		var cfs afero.Fs
 
 		if v.isResourceDir {
-			cfs = p.BaseFs.Resources.Fs
+			cfs = p.BaseFs.ResourcesCache
 		} else {
 			cfs = fs
 		}
 
-		var baseDir string
-		if !strings.HasPrefix(v.Dir, "_gen") {
-			// We do cache eviction (file removes) and since the user can set
-			// his/hers own cache directory, we really want to make sure
-			// we do not delete any files that do not belong to this cache.
-			// We do add the cache name as the root, but this is an extra safe
-			// guard. We skip the files inside /resources/_gen/ because
-			// that would be breaking.
-			baseDir = filepath.Join(v.Dir, filecacheRootDirname, k)
-		} else {
-			baseDir = filepath.Join(v.Dir, k)
-		}
-		if err = cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
+		baseDir := v.Dir
+
+		if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
 			return nil, err
 		}
 
 		bfs := afero.NewBasePathFs(cfs, baseDir)
 
-		m[k] = NewCache(bfs, v.MaxAge)
+		var pruneAllRootDir string
+		if k == cacheKeyModules {
+			pruneAllRootDir = "pkg"
+		}
+
+		m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir)
 	}
 
 	return m, nil
diff --git a/cache/filecache/filecache_config.go b/cache/filecache/filecache_config.go
index a6a0252b2..0c6b569c1 100644
--- a/cache/filecache/filecache_config.go
+++ b/cache/filecache/filecache_config.go
@@ -19,6 +19,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/config"
+
 	"github.com/gohugoio/hugo/helpers"
 
 	"github.com/mitchellh/mapstructure"
@@ -32,7 +34,7 @@ const (
 	resourcesGenDir = ":resourceDir/_gen"
 )
 
-var defaultCacheConfig = cacheConfig{
+var defaultCacheConfig = Config{
 	MaxAge: -1, // Never expire
 	Dir:    ":cacheDir/:project",
 }
@@ -42,9 +44,20 @@ const (
 	cacheKeyGetCSV  = "getcsv"
 	cacheKeyImages  = "images"
 	cacheKeyAssets  = "assets"
+	cacheKeyModules = "modules"
 )
 
-var defaultCacheConfigs = map[string]cacheConfig{
+type Configs map[string]Config
+
+func (c Configs) CacheDirModules() string {
+	return c[cacheKeyModules].Dir
+}
+
+var defaultCacheConfigs = Configs{
+	cacheKeyModules: {
+		MaxAge: -1,
+		Dir:    ":cacheDir/modules",
+	},
 	cacheKeyGetJSON: defaultCacheConfig,
 	cacheKeyGetCSV:  defaultCacheConfig,
 	cacheKeyImages: {
@@ -57,9 +70,7 @@ var defaultCacheConfigs = map[string]cacheConfig{
 	},
 }
 
-type cachesConfig map[string]cacheConfig
-
-type cacheConfig struct {
+type Config struct {
 	// Max age of cache entries in this cache. Any items older than this will
 	// be removed and not returned from the cache.
 	// a negative value means forever, 0 means cache is disabled.
@@ -88,13 +99,18 @@ func (f Caches) ImageCache() *Cache {
 	return f[cacheKeyImages]
 }
 
+// ModulesCache gets the file cache for Hugo Modules.
+func (f Caches) ModulesCache() *Cache {
+	return f[cacheKeyModules]
+}
+
 // AssetsCache gets the file cache for assets (processed resources, SCSS etc.).
 func (f Caches) AssetsCache() *Cache {
 	return f[cacheKeyAssets]
 }
 
-func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
-	c := make(cachesConfig)
+func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) {
+	c := make(Configs)
 	valid := make(map[string]bool)
 	// Add defaults
 	for k, v := range defaultCacheConfigs {
@@ -102,11 +118,9 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
 		valid[k] = true
 	}
 
-	cfg := p.Cfg
-
 	m := cfg.GetStringMap(cachesConfigKey)
 
-	_, isOsFs := p.Fs.Source.(*afero.OsFs)
+	_, isOsFs := fs.(*afero.OsFs)
 
 	for k, v := range m {
 		cc := defaultCacheConfig
@@ -148,7 +162,7 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
 
 		for i, part := range parts {
 			if strings.HasPrefix(part, ":") {
-				resolved, isResource, err := resolveDirPlaceholder(p, part)
+				resolved, isResource, err := resolveDirPlaceholder(fs, cfg, part)
 				if err != nil {
 					return c, err
 				}
@@ -176,6 +190,18 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
 			}
 		}
 
+		if !strings.HasPrefix(v.Dir, "_gen") {
+			// We do cache eviction (file removes) and since the user can set
+			// his/hers own cache directory, we really want to make sure
+			// we do not delete any files that do not belong to this cache.
+			// We do add the cache name as the root, but this is an extra safe
+			// guard. We skip the files inside /resources/_gen/ because
+			// that would be breaking.
+			v.Dir = filepath.Join(v.Dir, filecacheRootDirname, k)
+		} else {
+			v.Dir = filepath.Join(v.Dir, k)
+		}
+
 		if disabled {
 			v.MaxAge = 0
 		}
@@ -187,15 +213,17 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
 }
 
 // Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
-func resolveDirPlaceholder(p *helpers.PathSpec, placeholder string) (cacheDir string, isResource bool, err error) {
+func resolveDirPlaceholder(fs afero.Fs, cfg config.Provider, placeholder string) (cacheDir string, isResource bool, err error) {
+	workingDir := cfg.GetString("workingDir")
+
 	switch strings.ToLower(placeholder) {
 	case ":resourcedir":
 		return "", true, nil
 	case ":cachedir":
-		d, err := helpers.GetCacheDir(p.Fs.Source, p.Cfg)
+		d, err := helpers.GetCacheDir(fs, cfg)
 		return d, false, err
 	case ":project":
-		return filepath.Base(p.WorkingDir), false, nil
+		return filepath.Base(workingDir), false, nil
 	}
 
 	return "", false, errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
diff --git a/cache/filecache/filecache_config_test.go b/cache/filecache/filecache_config_test.go
index b0f5d2dc0..f2f75344b 100644
--- a/cache/filecache/filecache_config_test.go
+++ b/cache/filecache/filecache_config_test.go
@@ -20,10 +20,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gohugoio/hugo/helpers"
+	"github.com/spf13/afero"
 
 	"github.com/gohugoio/hugo/config"
-	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
@@ -57,22 +56,19 @@ dir = "/path/to/c3"
 
 	cfg, err := config.FromConfigString(configStr, "toml")
 	assert.NoError(err)
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
+	fs := afero.NewMemMapFs()
+	decoded, err := DecodeConfig(fs, cfg)
 	assert.NoError(err)
 
-	decoded, err := decodeConfig(p)
-	assert.NoError(err)
-
-	assert.Equal(4, len(decoded))
+	assert.Equal(5, len(decoded))
 
 	c2 := decoded["getcsv"]
 	assert.Equal("11h0m0s", c2.MaxAge.String())
-	assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir)
+	assert.Equal(filepath.FromSlash("/path/to/c2/filecache/getcsv"), c2.Dir)
 
 	c3 := decoded["images"]
 	assert.Equal(time.Duration(-1), c3.MaxAge)
-	assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir)
+	assert.Equal(filepath.FromSlash("/path/to/c3/filecache/images"), c3.Dir)
 
 }
 
@@ -105,14 +101,11 @@ dir = "/path/to/c3"
 
 	cfg, err := config.FromConfigString(configStr, "toml")
 	assert.NoError(err)
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
-	assert.NoError(err)
-
-	decoded, err := decodeConfig(p)
+	fs := afero.NewMemMapFs()
+	decoded, err := DecodeConfig(fs, cfg)
 	assert.NoError(err)
 
-	assert.Equal(4, len(decoded))
+	assert.Equal(5, len(decoded))
 
 	for _, v := range decoded {
 		assert.Equal(time.Duration(0), v.MaxAge)
@@ -133,24 +126,22 @@ func TestDecodeConfigDefault(t *testing.T) {
 		cfg.Set("cacheDir", "/cache/thecache")
 	}
 
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
-	assert.NoError(err)
+	fs := afero.NewMemMapFs()
 
-	decoded, err := decodeConfig(p)
+	decoded, err := DecodeConfig(fs, cfg)
 
 	assert.NoError(err)
 
-	assert.Equal(4, len(decoded))
+	assert.Equal(5, len(decoded))
 
 	imgConfig := decoded[cacheKeyImages]
 	jsonConfig := decoded[cacheKeyGetJSON]
 
 	if runtime.GOOS == "windows" {
-		assert.Equal("_gen", imgConfig.Dir)
+		assert.Equal(filepath.FromSlash("_gen/images"), imgConfig.Dir)
 	} else {
-		assert.Equal("_gen", imgConfig.Dir)
-		assert.Equal("/cache/thecache/hugoproject", jsonConfig.Dir)
+		assert.Equal("_gen/images", imgConfig.Dir)
+		assert.Equal("/cache/thecache/hugoproject/filecache/getjson", jsonConfig.Dir)
 	}
 
 	assert.True(imgConfig.isResourceDir)
@@ -183,11 +174,9 @@ dir = "/"
 
 	cfg, err := config.FromConfigString(configStr, "toml")
 	assert.NoError(err)
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
-	assert.NoError(err)
+	fs := afero.NewMemMapFs()
 
-	_, err = decodeConfig(p)
+	_, err = DecodeConfig(fs, cfg)
 	assert.Error(err)
 
 }
diff --git a/cache/filecache/filecache_pruner.go b/cache/filecache/filecache_pruner.go
index 322eabf92..c6fd4497e 100644
--- a/cache/filecache/filecache_pruner.go
+++ b/cache/filecache/filecache_pruner.go
@@ -28,53 +28,100 @@ import (
 func (c Caches) Prune() (int, error) {
 	counter := 0
 	for k, cache := range c {
-		err := afero.Walk(cache.Fs, "", func(name string, info os.FileInfo, err error) error {
-			if info == nil {
-				return nil
-			}
 
-			name = cleanID(name)
-
-			if info.IsDir() {
-				f, err := cache.Fs.Open(name)
-				if err != nil {
-					// This cache dir may not exist.
-					return nil
-				}
-				defer f.Close()
-				_, err = f.Readdirnames(1)
-				if err == io.EOF {
-					// Empty dir.
-					return cache.Fs.Remove(name)
-				}
+		count, err := cache.Prune(false)
+
+		if err != nil {
+			return counter, errors.Wrapf(err, "failed to prune cache %q", k)
+		}
+
+		counter += count
+
+	}
+
+	return counter, nil
+}
+
+// Prune removes expired and unused items from this cache.
+// If force is set, everything will be removed not considering expiry time.
+func (c *Cache) Prune(force bool) (int, error) {
+	if c.pruneAllRootDir != "" {
+		return c.pruneRootDir(force)
+	}
+
+	counter := 0
+
+	err := afero.Walk(c.Fs, "", func(name string, info os.FileInfo, err error) error {
+		if info == nil {
+			return nil
+		}
+
+		name = cleanID(name)
 
+		if info.IsDir() {
+			f, err := c.Fs.Open(name)
+			if err != nil {
+				// This cache dir may not exist.
 				return nil
 			}
+			defer f.Close()
+			_, err = f.Readdirnames(1)
+			if err == io.EOF {
+				// Empty dir.
+				return c.Fs.Remove(name)
+			}
+
+			return nil
+		}
 
-			shouldRemove := cache.isExpired(info.ModTime())
+		shouldRemove := force || c.isExpired(info.ModTime())
 
-			if !shouldRemove && len(cache.nlocker.seen) > 0 {
-				// Remove it if it's not been touched/used in the last build.
-				_, seen := cache.nlocker.seen[name]
-				shouldRemove = !seen
-			}
+		if !shouldRemove && len(c.nlocker.seen) > 0 {
+			// Remove it if it's not been touched/used in the last build.
+			_, seen := c.nlocker.seen[name]
+			shouldRemove = !seen
+		}
 
-			if shouldRemove {
-				err := cache.Fs.Remove(name)
-				if err == nil {
-					counter++
-				}
-				return err
+		if shouldRemove {
+			err := c.Fs.Remove(name)
+			if err == nil {
+				counter++
 			}
+			return err
+		}
 
-			return nil
-		})
+		return nil
+	})
 
-		if err != nil {
-			return counter, errors.Wrapf(err, "failed to prune cache %q", k)
+	return counter, err
+}
+
+func (c *Cache) pruneRootDir(force bool) (int, error) {
+
+	info, err := c.Fs.Stat(c.pruneAllRootDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return 0, nil
 		}
+		return 0, err
+	}
 
+	if !force && !c.isExpired(info.ModTime()) {
+		return 0, nil
 	}
 
-	return counter, nil
+	counter := 0
+	// Module cache has 0555 directories; make them writable in order to remove content.
+	afero.Walk(c.Fs, c.pruneAllRootDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if info.IsDir() {
+			counter++
+			c.Fs.Chmod(path, 0777)
+		}
+		return nil
+	})
+	return 1, c.Fs.RemoveAll(c.pruneAllRootDir)
+
 }
diff --git a/cache/filecache/filecache_pruner_test.go b/cache/filecache/filecache_pruner_test.go
index e62a6315a..72c6781ac 100644
--- a/cache/filecache/filecache_pruner_test.go
+++ b/cache/filecache/filecache_pruner_test.go
@@ -18,9 +18,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gohugoio/hugo/config"
-	"github.com/gohugoio/hugo/helpers"
-	"github.com/gohugoio/hugo/hugofs"
+	"github.com/spf13/afero"
 
 	"github.com/stretchr/testify/require"
 )
@@ -54,14 +52,9 @@ maxAge = "200ms"
 dir = ":resourceDir/_gen"
 `
 
-	cfg, err := config.FromConfigString(configStr, "toml")
-	assert.NoError(err)
-
 	for _, name := range []string{cacheKeyGetCSV, cacheKeyGetJSON, cacheKeyAssets, cacheKeyImages} {
 		msg := fmt.Sprintf("cache: %s", name)
-		fs := hugofs.NewMem(cfg)
-		p, err := helpers.NewPathSpec(fs, cfg)
-		assert.NoError(err)
+		p := newPathsSpec(t, afero.NewMemMapFs(), configStr)
 		caches, err := NewCaches(p)
 		assert.NoError(err)
 		cache := caches[name]
diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go
index 5ac2e9beb..a03c3116a 100644
--- a/cache/filecache/filecache_test.go
+++ b/cache/filecache/filecache_test.go
@@ -25,6 +25,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gohugoio/hugo/langs"
+	"github.com/gohugoio/hugo/modules"
+
 	"github.com/gohugoio/hugo/common/hugio"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
@@ -83,12 +86,7 @@ dir = ":cacheDir/c"
 		configStr = replacer.Replace(configStr)
 		configStr = strings.Replace(configStr, "\\", winPathSep, -1)
 
-		cfg, err := config.FromConfigString(configStr, "toml")
-		assert.NoError(err)
-
-		fs := hugofs.NewFrom(osfs, cfg)
-		p, err := helpers.NewPathSpec(fs, cfg)
-		assert.NoError(err)
+		p := newPathsSpec(t, osfs, configStr)
 
 		caches, err := NewCaches(p)
 		assert.NoError(err)
@@ -207,11 +205,7 @@ dir = "/cache/c"
 
 `
 
-	cfg, err := config.FromConfigString(configStr, "toml")
-	assert.NoError(err)
-	fs := hugofs.NewMem(cfg)
-	p, err := helpers.NewPathSpec(fs, cfg)
-	assert.NoError(err)
+	p := newPathsSpec(t, afero.NewMemMapFs(), configStr)
 
 	caches, err := NewCaches(p)
 	assert.NoError(err)
@@ -255,3 +249,51 @@ func TestCleanID(t *testing.T) {
 	assert.Equal(filepath.FromSlash("a/b/c.txt"), cleanID(filepath.FromSlash("/a/b//c.txt")))
 	assert.Equal(filepath.FromSlash("a/b/c.txt"), cleanID(filepath.FromSlash("a/b//c.txt")))
 }
+
+func initConfig(fs afero.Fs, cfg config.Provider) error {
+	if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil {
+		return err
+	}
+
+	modConfig, err := modules.DecodeConfig(cfg)
+	if err != nil {
+		return err
+	}
+
+	workingDir := cfg.GetString("workingDir")
+	themesDir := cfg.GetString("themesDir")
+	if !filepath.IsAbs(themesDir) {
+		themesDir = filepath.Join(workingDir, themesDir)
+	}
+	modulesClient := modules.NewClient(modules.ClientConfig{
+		Fs:           fs,
+		WorkingDir:   workingDir,
+		ThemesDir:    themesDir,
+		ModuleConfig: modConfig,
+		IgnoreVendor: true,
+	})
+
+	moduleConfig, err := modulesClient.Collect()
+	if err != nil {
+		return err
+	}
+
+	if err := modules.ApplyProjectConfigDefaults(cfg, moduleConfig.ActiveModules[len(moduleConfig.ActiveModules)-1]); err != nil {
+		return err
+	}
+
+	cfg.Set("allModules", moduleConfig.ActiveModules)
+
+	return nil
+}
+
+func newPathsSpec(t *testing.T, fs afero.Fs, configStr string) *helpers.PathSpec {
+	assert := require.New(t)
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+	initConfig(fs, cfg)
+	p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg)
+	assert.NoError(err)
+	return p
+
+}