Add Hugo Modules

This commit implements Hugo Modules.

This is a broad subject, but some keywords include:

* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`,  `hugo mod get`,  `hugo mod graph`,  `hugo mod tidy`, and  `hugo mod vendor`.

All of the above is backed by Go Modules.

Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092

1 files changed, 570 insertions, 0 deletions

diff --git a/modules/client.go b/modules/client.go
new file mode 100644
index 000000000..ac09721dc
--- /dev/null
+++ b/modules/client.go
@@ -0,0 +1,570 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/gohugoio/hugo/hugofs/files"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"strings"
+	"time"
+
+	"github.com/gohugoio/hugo/config"
+
+	"github.com/rogpeppe/go-internal/module"
+
+	"github.com/gohugoio/hugo/common/hugio"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/afero"
+)
+
+var (
+	fileSeparator = string(os.PathSeparator)
+)
+
+const (
+	goBinaryStatusOK goBinaryStatus = iota
+	goBinaryStatusNotFound
+	goBinaryStatusTooOld
+)
+
+// The "vendor" dir is reserved for Go Modules.
+const vendord = "_vendor"
+
+const (
+	goModFilename = "go.mod"
+	goSumFilename = "go.sum"
+)
+
+// NewClient creates a new Client that can be used to manage the Hugo Components
+// in a given workingDir.
+// The Client will resolve the dependencies recursively, but needs the top
+// level imports to start out.
+func NewClient(cfg ClientConfig) *Client {
+	fs := cfg.Fs
+
+	n := filepath.Join(cfg.WorkingDir, goModFilename)
+	goModEnabled, _ := afero.Exists(fs, n)
+	var goModFilename string
+	if goModEnabled {
+		goModFilename = n
+	}
+
+	env := os.Environ()
+	mcfg := cfg.ModuleConfig
+
+	config.SetEnvVars(&env,
+		"PWD", cfg.WorkingDir,
+		"GOPROXY", mcfg.Proxy,
+		"GOPRIVATE", mcfg.Private,
+		"GONOPROXY", mcfg.NoProxy)
+
+	if cfg.CacheDir != "" {
+		// Module cache stored below $GOPATH/pkg
+		config.SetEnvVars(&env, "GOPATH", cfg.CacheDir)
+
+	}
+
+	logger := cfg.Logger
+	if logger == nil {
+		logger = loggers.NewWarningLogger()
+	}
+
+	return &Client{
+		fs:                fs,
+		ignoreVendor:      cfg.IgnoreVendor,
+		workingDir:        cfg.WorkingDir,
+		themesDir:         cfg.ThemesDir,
+		logger:            logger,
+		moduleConfig:      mcfg,
+		environ:           env,
+		GoModulesFilename: goModFilename}
+}
+
+// Client contains most of the API provided by this package.
+type Client struct {
+	fs     afero.Fs
+	logger *loggers.Logger
+
+	// Ignore any _vendor directory.
+	ignoreVendor bool
+
+	// Absolute path to the project dir.
+	workingDir string
+
+	// Absolute path to the project's themes dir.
+	themesDir string
+
+	// The top level module config
+	moduleConfig Config
+
+	// Environment variables used in "go get" etc.
+	environ []string
+
+	// Set when Go modules are initialized in the current repo, that is:
+	// a go.mod file exists.
+	GoModulesFilename string
+
+	// Set if we get a exec.ErrNotFound when running Go, which is most likely
+	// due to being run on a system without Go installed. We record it here
+	// so we can give an instructional error at the end if module/theme
+	// resolution fails.
+	goBinaryStatus goBinaryStatus
+}
+
+// Graph writes a module dependenchy graph to the given writer.
+func (c *Client) Graph(w io.Writer) error {
+	mc, coll := c.collect(true)
+	if coll.err != nil {
+		return coll.err
+	}
+	for _, module := range mc.AllModules {
+		if module.Owner() == nil {
+			continue
+		}
+
+		prefix := ""
+		if module.Disabled() {
+			prefix = "DISABLED "
+		}
+		dep := pathVersion(module.Owner()) + " " + pathVersion(module)
+		if replace := module.Replace(); replace != nil {
+			if replace.Version() != "" {
+				dep += " => " + pathVersion(replace)
+			} else {
+				// Local dir.
+				dep += " => " + replace.Dir()
+			}
+
+		}
+		fmt.Fprintln(w, prefix+dep)
+	}
+
+	return nil
+}
+
+// Tidy can be used to remove unused dependencies from go.mod and go.sum.
+func (c *Client) Tidy() error {
+	tc, coll := c.collect(false)
+	if coll.err != nil {
+		return coll.err
+	}
+
+	if coll.skipTidy {
+		return nil
+	}
+
+	return c.tidy(tc.AllModules, false)
+}
+
+// Vendor writes all the module dependencies to a _vendor folder.
+//
+// Unlike Go, we support it for any level.
+//
+// We, by default, use the /_vendor folder first, if found. To disable,
+// run with
+//    hugo --ignoreVendor
+//
+// Given a module tree, Hugo will pick the first module for a given path,
+// meaning that if the top-level module is vendored, that will be the full
+// set of dependencies.
+func (c *Client) Vendor() error {
+	vendorDir := filepath.Join(c.workingDir, vendord)
+	if err := c.rmVendorDir(vendorDir); err != nil {
+		return err
+	}
+
+	// Write the modules list to modules.txt.
+	//
+	// On the form:
+	//
+	// # github.com/alecthomas/chroma v0.6.3
+	//
+	// This is how "go mod vendor" does it. Go also lists
+	// the packages below it, but that is currently not applicable to us.
+	//
+	var modulesContent bytes.Buffer
+
+	tc, coll := c.collect(true)
+	if coll.err != nil {
+		return coll.err
+	}
+
+	for _, t := range tc.AllModules {
+		if t.Owner() == nil {
+			// This is the project.
+			continue
+		}
+		// We respect the --ignoreVendor flag even for the vendor command.
+		if !t.IsGoMod() && !t.Vendor() {
+			// We currently do not vendor components living in the
+			// theme directory, see https://github.com/gohugoio/hugo/issues/5993
+			continue
+		}
+
+		fmt.Fprintln(&modulesContent, "# "+t.Path()+" "+t.Version())
+
+		dir := t.Dir()
+
+		for _, mount := range t.Mounts() {
+			if err := hugio.CopyDir(c.fs, filepath.Join(dir, mount.Source), filepath.Join(vendorDir, t.Path(), mount.Source), nil); err != nil {
+				return errors.Wrap(err, "failed to copy module to vendor dir")
+			}
+		}
+
+		// Include the resource cache if present.
+		resourcesDir := filepath.Join(dir, files.FolderResources)
+		_, err := c.fs.Stat(resourcesDir)
+		if err == nil {
+			if err := hugio.CopyDir(c.fs, resourcesDir, filepath.Join(vendorDir, t.Path(), files.FolderResources), nil); err != nil {
+				return errors.Wrap(err, "failed to copy resources to vendor dir")
+			}
+		}
+
+		// Also include any theme.toml or config.* files in the root.
+		configFiles, _ := afero.Glob(c.fs, filepath.Join(dir, "config.*"))
+		configFiles = append(configFiles, filepath.Join(dir, "theme.toml"))
+		for _, configFile := range configFiles {
+			if err := hugio.CopyFile(c.fs, configFile, filepath.Join(vendorDir, t.Path(), filepath.Base(configFile))); err != nil {
+				if !os.IsNotExist(err) {
+					return err
+				}
+			}
+		}
+	}
+
+	if modulesContent.Len() > 0 {
+		if err := afero.WriteFile(c.fs, filepath.Join(vendorDir, vendorModulesFilename), modulesContent.Bytes(), 0666); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Get runs "go get" with the supplied arguments.
+func (c *Client) Get(args ...string) error {
+	if err := c.runGo(context.Background(), os.Stdout, append([]string{"get"}, args...)...); err != nil {
+		errors.Wrapf(err, "failed to get %q", args)
+	}
+	return nil
+}
+
+// Init initializes this as a Go Module with the given path.
+// If path is empty, Go will try to guess.
+// If this succeeds, this project will be marked as Go Module.
+func (c *Client) Init(path string) error {
+	err := c.runGo(context.Background(), os.Stdout, "mod", "init", path)
+	if err != nil {
+		return errors.Wrap(err, "failed to init modules")
+	}
+
+	c.GoModulesFilename = filepath.Join(c.workingDir, goModFilename)
+
+	return nil
+}
+
+func (c *Client) isProbablyModule(path string) bool {
+	return module.CheckPath(path) == nil
+}
+
+func (c *Client) listGoMods() (goModules, error) {
+	if c.GoModulesFilename == "" {
+		return nil, nil
+	}
+
+	out := ioutil.Discard
+	err := c.runGo(context.Background(), out, "mod", "download")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to download modules")
+	}
+
+	b := &bytes.Buffer{}
+	err = c.runGo(context.Background(), b, "list", "-m", "-json", "all")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to list modules")
+	}
+
+	var modules goModules
+
+	dec := json.NewDecoder(b)
+	for {
+		m := &goModule{}
+		if err := dec.Decode(m); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, errors.Wrap(err, "failed to decode modules list")
+		}
+
+		modules = append(modules, m)
+	}
+
+	return modules, err
+
+}
+
+func (c *Client) rewriteGoMod(name string, isGoMod map[string]bool) error {
+	data, err := c.rewriteGoModRewrite(name, isGoMod)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := afero.WriteFile(c.fs, filepath.Join(c.workingDir, name), data, 0666); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *Client) rewriteGoModRewrite(name string, isGoMod map[string]bool) ([]byte, error) {
+	if name == goModFilename && c.GoModulesFilename == "" {
+		// Already checked.
+		return nil, nil
+	}
+
+	modlineSplitter := getModlineSplitter(name == goModFilename)
+
+	b := &bytes.Buffer{}
+	f, err := c.fs.Open(filepath.Join(c.workingDir, name))
+	if err != nil {
+		if os.IsNotExist(err) {
+			// It's been deleted.
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	var dirty bool
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		var doWrite bool
+
+		if parts := modlineSplitter(line); parts != nil {
+			modname, modver := parts[0], parts[1]
+			modver = strings.TrimSuffix(modver, "/"+goModFilename)
+			modnameVer := modname + " " + modver
+			doWrite = isGoMod[modnameVer]
+		} else {
+			doWrite = true
+		}
+
+		if doWrite {
+			fmt.Fprintln(b, line)
+		} else {
+			dirty = true
+		}
+	}
+
+	if !dirty {
+		// Nothing changed
+		return nil, nil
+	}
+
+	return b.Bytes(), nil
+
+}
+
+func (c *Client) rmVendorDir(vendorDir string) error {
+	modulestxt := filepath.Join(vendorDir, vendorModulesFilename)
+
+	if _, err := c.fs.Stat(vendorDir); err != nil {
+		return nil
+	}
+
+	_, err := c.fs.Stat(modulestxt)
+	if err != nil {
+		// If we have a _vendor dir without modules.txt it sounds like
+		// a _vendor dir created by others.
+		return errors.New("found _vendor dir without modules.txt, skip delete")
+	}
+
+	return c.fs.RemoveAll(vendorDir)
+}
+
+func (c *Client) runGo(
+	ctx context.Context,
+	stdout io.Writer,
+	args ...string) error {
+
+	if c.goBinaryStatus != 0 {
+		return nil
+	}
+
+	stderr := new(bytes.Buffer)
+	cmd := exec.CommandContext(ctx, "go", args...)
+
+	cmd.Env = c.environ
+	cmd.Dir = c.workingDir
+	cmd.Stdout = stdout
+	cmd.Stderr = io.MultiWriter(stderr, os.Stderr)
+
+	if err := cmd.Run(); err != nil {
+		if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound {
+			c.goBinaryStatus = goBinaryStatusNotFound
+			return nil
+		}
+
+		_, ok := err.(*exec.ExitError)
+		if !ok {
+			return errors.Errorf("failed to execute 'go %v': %s %T", args, err, err)
+		}
+
+		// Too old Go version
+		if strings.Contains(stderr.String(), "flag provided but not defined") {
+			c.goBinaryStatus = goBinaryStatusTooOld
+			return nil
+		}
+
+		return errors.Errorf("go command failed: %s", stderr)
+
+	}
+
+	return nil
+}
+
+func (c *Client) tidy(mods Modules, goModOnly bool) error {
+	isGoMod := make(map[string]bool)
+	for _, m := range mods {
+		if m.Owner() == nil {
+			continue
+		}
+		if m.IsGoMod() {
+			// Matching the format in go.mod
+			pathVer := m.Path() + " " + m.Version()
+			isGoMod[pathVer] = true
+		}
+	}
+
+	if err := c.rewriteGoMod(goModFilename, isGoMod); err != nil {
+		return err
+	}
+
+	if goModOnly {
+		return nil
+	}
+
+	if err := c.rewriteGoMod(goSumFilename, isGoMod); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ClientConfig configures the module Client.
+type ClientConfig struct {
+	Fs           afero.Fs
+	Logger       *loggers.Logger
+	IgnoreVendor bool
+	WorkingDir   string
+	ThemesDir    string // Absolute directory path
+	CacheDir     string // Module cache
+	ModuleConfig Config
+}
+
+type goBinaryStatus int
+
+type goModule struct {
+	Path     string         // module path
+	Version  string         // module version
+	Versions []string       // available module versions (with -versions)
+	Replace  *goModule      // replaced by this module
+	Time     *time.Time     // time version was created
+	Update   *goModule      // available update, if any (with -u)
+	Main     bool           // is this the main module?
+	Indirect bool           // is this module only an indirect dependency of main module?
+	Dir      string         // directory holding files for this module, if any
+	GoMod    string         // path to go.mod file for this module, if any
+	Error    *goModuleError // error loading module
+}
+
+type goModuleError struct {
+	Err string // the error itself
+}
+
+type goModules []*goModule
+
+func (modules goModules) GetByPath(p string) *goModule {
+	if modules == nil {
+		return nil
+	}
+
+	for _, m := range modules {
+		if strings.EqualFold(p, m.Path) {
+			return m
+		}
+	}
+
+	return nil
+}
+
+func (modules goModules) GetMain() *goModule {
+	for _, m := range modules {
+		if m.Main {
+			return m
+		}
+	}
+
+	return nil
+}
+
+func getModlineSplitter(isGoMod bool) func(line string) []string {
+	if isGoMod {
+		return func(line string) []string {
+			if strings.HasPrefix(line, "require (") {
+				return nil
+			}
+			if !strings.HasPrefix(line, "require") && !strings.HasPrefix(line, "\t") {
+				return nil
+			}
+			line = strings.TrimPrefix(line, "require")
+			line = strings.TrimSpace(line)
+			line = strings.TrimSuffix(line, "// indirect")
+
+			return strings.Fields(line)
+		}
+	}
+
+	return func(line string) []string {
+		return strings.Fields(line)
+	}
+}
+
+func pathVersion(m Module) string {
+	versionStr := m.Version()
+	if m.Vendor() {
+		versionStr += "+vendor"
+	}
+	if versionStr == "" {
+		return m.Path()
+	}
+	return fmt.Sprintf("%s@%s", m.Path(), versionStr)
+}