Add Hugo Modules

This commit implements Hugo Modules.

This is a broad subject, but some keywords include:

* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`,  `hugo mod get`,  `hugo mod graph`,  `hugo mod tidy`, and  `hugo mod vendor`.

All of the above is backed by Go Modules.

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092

4 files changed, 279 insertions, 32 deletions

diff --git a/langs/config.go b/langs/config.go
new file mode 100644
index 000000000..927f3558f
--- /dev/null
+++ b/langs/config.go
@@ -0,0 +1,217 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package langs
+
+import (
+	"fmt"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/gohugoio/hugo/common/maps"
+
+	"github.com/spf13/cast"
+
+	"github.com/pkg/errors"
+
+	"github.com/gohugoio/hugo/config"
+)
+
+type LanguagesConfig struct {
+	Languages                      Languages
+	Multihost                      bool
+	DefaultContentLanguageInSubdir bool
+}
+
+func LoadLanguageSettings(cfg config.Provider, oldLangs Languages) (c LanguagesConfig, err error) {
+
+	defaultLang := cfg.GetString("defaultContentLanguage")
+	if defaultLang == "" {
+		defaultLang = "en"
+		cfg.Set("defaultContentLanguage", defaultLang)
+	}
+
+	var languages map[string]interface{}
+
+	languagesFromConfig := cfg.GetStringMap("languages")
+	disableLanguages := cfg.GetStringSlice("disableLanguages")
+
+	if len(disableLanguages) == 0 {
+		languages = languagesFromConfig
+	} else {
+		languages = make(map[string]interface{})
+		for k, v := range languagesFromConfig {
+			for _, disabled := range disableLanguages {
+				if disabled == defaultLang {
+					return c, fmt.Errorf("cannot disable default language %q", defaultLang)
+				}
+
+				if strings.EqualFold(k, disabled) {
+					v.(map[string]interface{})["disabled"] = true
+					break
+				}
+			}
+			languages[k] = v
+		}
+	}
+
+	var languages2 Languages
+
+	if len(languages) == 0 {
+		languages2 = append(languages2, NewDefaultLanguage(cfg))
+	} else {
+		languages2, err = toSortedLanguages(cfg, languages)
+		if err != nil {
+			return c, errors.Wrap(err, "Failed to parse multilingual config")
+		}
+	}
+
+	if oldLangs != nil {
+		// When in multihost mode, the languages are mapped to a server, so
+		// some structural language changes will need a restart of the dev server.
+		// The validation below isn't complete, but should cover the most
+		// important cases.
+		var invalid bool
+		if languages2.IsMultihost() != oldLangs.IsMultihost() {
+			invalid = true
+		} else {
+			if languages2.IsMultihost() && len(languages2) != len(oldLangs) {
+				invalid = true
+			}
+		}
+
+		if invalid {
+			return c, errors.New("language change needing a server restart detected")
+		}
+
+		if languages2.IsMultihost() {
+			// We need to transfer any server baseURL to the new language
+			for i, ol := range oldLangs {
+				nl := languages2[i]
+				nl.Set("baseURL", ol.GetString("baseURL"))
+			}
+		}
+	}
+
+	// The defaultContentLanguage is something the user has to decide, but it needs
+	// to match a language in the language definition list.
+	langExists := false
+	for _, lang := range languages2 {
+		if lang.Lang == defaultLang {
+			langExists = true
+			break
+		}
+	}
+
+	if !langExists {
+		return c, fmt.Errorf("site config value %q for defaultContentLanguage does not match any language definition", defaultLang)
+	}
+
+	c.Languages = languages2
+	c.Multihost = languages2.IsMultihost()
+	c.DefaultContentLanguageInSubdir = c.Multihost
+
+	sortedDefaultFirst := make(Languages, len(c.Languages))
+	for i, v := range c.Languages {
+		sortedDefaultFirst[i] = v
+	}
+	sort.Slice(sortedDefaultFirst, func(i, j int) bool {
+		li, lj := sortedDefaultFirst[i], sortedDefaultFirst[j]
+		if li.Lang == defaultLang {
+			return true
+		}
+
+		if lj.Lang == defaultLang {
+			return false
+		}
+
+		return i < j
+	})
+
+	cfg.Set("languagesSorted", c.Languages)
+	cfg.Set("languagesSortedDefaultFirst", sortedDefaultFirst)
+	cfg.Set("multilingual", len(languages2) > 1)
+
+	multihost := c.Multihost
+
+	if multihost {
+		cfg.Set("defaultContentLanguageInSubdir", true)
+		cfg.Set("multihost", true)
+	}
+
+	if multihost {
+		// The baseURL may be provided at the language level. If that is true,
+		// then every language must have a baseURL. In this case we always render
+		// to a language sub folder, which is then stripped from all the Permalink URLs etc.
+		for _, l := range languages2 {
+			burl := l.GetLocal("baseURL")
+			if burl == nil {
+				return c, errors.New("baseURL must be set on all or none of the languages")
+			}
+		}
+
+	}
+
+	return c, nil
+}
+
+func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (Languages, error) {
+	languages := make(Languages, len(l))
+	i := 0
+
+	for lang, langConf := range l {
+		langsMap, err := cast.ToStringMapE(langConf)
+
+		if err != nil {
+			return nil, fmt.Errorf("Language config is not a map: %T", langConf)
+		}
+
+		language := NewLanguage(lang, cfg)
+
+		for loki, v := range langsMap {
+			switch loki {
+			case "title":
+				language.Title = cast.ToString(v)
+			case "languagename":
+				language.LanguageName = cast.ToString(v)
+			case "weight":
+				language.Weight = cast.ToInt(v)
+			case "contentdir":
+				language.ContentDir = filepath.Clean(cast.ToString(v))
+			case "disabled":
+				language.Disabled = cast.ToBool(v)
+			case "params":
+				m := cast.ToStringMap(v)
+				// Needed for case insensitive fetching of params values
+				maps.ToLower(m)
+				for k, vv := range m {
+					language.SetParam(k, vv)
+				}
+			}
+
+			// Put all into the Params map
+			language.SetParam(loki, v)
+
+			// Also set it in the configuration map (for baseURL etc.)
+			language.Set(loki, v)
+		}
+
+		languages[i] = language
+		i++
+	}
+
+	sort.Sort(languages)
+
+	return languages, nil
+}
diff --git a/langs/i18n/i18n_test.go b/langs/i18n/i18n_test.go
index b67cabc55..e08210848 100644
--- a/langs/i18n/i18n_test.go
+++ b/langs/i18n/i18n_test.go
@@ -17,11 +17,13 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/gohugoio/hugo/modules"
+
 	"github.com/gohugoio/hugo/tpl/tplimpl"
 
 	"github.com/gohugoio/hugo/common/loggers"
-	"github.com/gohugoio/hugo/htesting"
 	"github.com/gohugoio/hugo/langs"
+	"github.com/gohugoio/hugo/resources/page"
 	"github.com/spf13/afero"
 	"github.com/spf13/viper"
 
@@ -199,7 +201,7 @@ func newDepsConfig(tp *TranslationProvider, cfg config.Provider, fs *hugofs.Fs)
 	l.Set("i18nDir", "i18n")
 	return deps.DepsCfg{
 		Language:            l,
-		Site:                htesting.NewTestHugoSite(),
+		Site:                page.NewDummyHugoSite(cfg),
 		Cfg:                 cfg,
 		Fs:                  fs,
 		Logger:              logger,
@@ -219,6 +221,13 @@ func getConfig() *viper.Viper {
 	v.Set("assetDir", "assets")
 	v.Set("resourceDir", "resources")
 	v.Set("publishDir", "public")
+	langs.LoadLanguageSettings(v, nil)
+	mod, err := modules.CreateProjectModule(v)
+	if err != nil {
+		panic(err)
+	}
+	v.Set("allModules", modules.Modules{mod})
+
 	return v
 
 }
diff --git a/langs/i18n/translationProvider.go b/langs/i18n/translationProvider.go
index 74e144007..c7b4839ee 100644
--- a/langs/i18n/translationProvider.go
+++ b/langs/i18n/translationProvider.go
@@ -40,8 +40,7 @@ func NewTranslationProvider() *TranslationProvider {
 
 // Update updates the i18n func in the provided Deps.
 func (tp *TranslationProvider) Update(d *deps.Deps) error {
-	sp := source.NewSourceSpec(d.PathSpec, d.BaseFs.SourceFilesystems.I18n.Fs)
-	src := sp.NewFilesystem("")
+	spec := source.NewSourceSpec(d.PathSpec, nil)
 
 	i18nBundle := bundle.New()
 
@@ -51,25 +50,33 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error {
 	}
 	var newLangs []string
 
-	for _, r := range src.Files() {
-		currentSpec := language.GetPluralSpec(r.BaseFileName())
-		if currentSpec == nil {
-			// This may is a language code not supported by go-i18n, it may be
-			// Klingon or ... not even a fake language. Make sure it works.
-			newLangs = append(newLangs, r.BaseFileName())
+	for _, dir := range d.BaseFs.I18n.Dirs {
+		src := spec.NewFilesystemFromFileMetaInfo(dir)
+
+		files, err := src.Files()
+		if err != nil {
+			return err
 		}
-	}
 
-	if len(newLangs) > 0 {
-		language.RegisterPluralSpec(newLangs, en)
-	}
+		for _, r := range files {
+			currentSpec := language.GetPluralSpec(r.BaseFileName())
+			if currentSpec == nil {
+				// This may is a language code not supported by go-i18n, it may be
+				// Klingon or ... not even a fake language. Make sure it works.
+				newLangs = append(newLangs, r.BaseFileName())
+			}
+		}
 
-	// The source files are ordered so the most important comes first. Since this is a
-	// last key win situation, we have to reverse the iteration order.
-	files := src.Files()
-	for i := len(files) - 1; i >= 0; i-- {
-		if err := addTranslationFile(i18nBundle, files[i]); err != nil {
-			return err
+		if len(newLangs) > 0 {
+			language.RegisterPluralSpec(newLangs, en)
+		}
+
+		// The source files are ordered so the most important comes first. Since this is a
+		// last key win situation, we have to reverse the iteration order.
+		for i := len(files) - 1; i >= 0; i-- {
+			if err := addTranslationFile(i18nBundle, files[i]); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -81,8 +88,8 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error {
 
 }
 
-func addTranslationFile(bundle *bundle.Bundle, r source.ReadableFile) error {
-	f, err := r.Open()
+func addTranslationFile(bundle *bundle.Bundle, r source.File) error {
+	f, err := r.FileInfo().Meta().Open()
 	if err != nil {
 		return _errors.Wrapf(err, "failed to open translations file %q:", r.LogicalName())
 	}
@@ -101,14 +108,15 @@ func (tp *TranslationProvider) Clone(d *deps.Deps) error {
 	return nil
 }
 
-func errWithFileContext(inerr error, r source.ReadableFile) error {
-	rfi, ok := r.FileInfo().(hugofs.RealFilenameInfo)
+func errWithFileContext(inerr error, r source.File) error {
+	fim, ok := r.FileInfo().(hugofs.FileMetaInfo)
 	if !ok {
 		return inerr
 	}
 
-	realFilename := rfi.RealFilename()
-	f, err := r.Open()
+	meta := fim.Meta()
+	realFilename := meta.Filename()
+	f, err := meta.Open()
 	if err != nil {
 		return inerr
 	}
diff --git a/langs/language.go b/langs/language.go
index 14e3263ae..f71b0255b 100644
--- a/langs/language.go
+++ b/langs/language.go
@@ -78,12 +78,7 @@ func NewLanguage(lang string, cfg config.Provider) *Language {
 	}
 	maps.ToLower(params)
 
-	defaultContentDir := cfg.GetString("contentDir")
-	if defaultContentDir == "" {
-		panic("contentDir not set")
-	}
-
-	l := &Language{Lang: lang, ContentDir: defaultContentDir, Cfg: cfg, params: params, settings: make(map[string]interface{})}
+	l := &Language{Lang: lang, ContentDir: cfg.GetString("contentDir"), Cfg: cfg, params: params, settings: make(map[string]interface{})}
 	return l
 }
 
@@ -132,6 +127,24 @@ func (l *Language) Params() map[string]interface{} {
 	return l.params
 }
 
+func (l Languages) AsSet() map[string]bool {
+	m := make(map[string]bool)
+	for _, lang := range l {
+		m[lang.Lang] = true
+	}
+
+	return m
+}
+
+func (l Languages) AsOrdinalSet() map[string]int {
+	m := make(map[string]int)
+	for i, lang := range l {
+		m[lang.Lang] = i
+	}
+
+	return m
+}
+
 // IsMultihost returns whether there are more than one language and at least one of
 // the languages has baseURL specificed on the language level.
 func (l Languages) IsMultihost() bool {