diff options
author | Bjørn Erik Pedersen <[email protected]> | 2018-03-01 15:01:25 +0100 |
---|---|---|
committer | Bjørn Erik Pedersen <[email protected]> | 2018-06-10 23:55:20 +0200 |
commit | 80230f26a3020ff33bac2bef01b2c0e314b89f86 (patch) | |
tree | 6da3114350477866065d265d6e1db9cb55639dc1 | |
parent | 6464981adb4d7d0f41e8e2c987342082982210a1 (diff) | |
download | hugo-80230f26a3020ff33bac2bef01b2c0e314b89f86.tar.gz hugo-80230f26a3020ff33bac2bef01b2c0e314b89f86.zip |
Add support for theme composition and inheritance
This commit adds support for theme composition and inheritance in Hugo.
With this, it helps thinking about a theme as a set of ordered components:
```toml
theme = ["my-shortcodes", "base-theme", "hyde"]
```
The theme definition example above in `config.toml` creates a theme with the 3 components with presedence from left to right.
So, Hugo will, for any given file, data entry etc., look first in the project, and then in `my-shortcode`, `base-theme` and lastly `hyde`.
Hugo uses two different algorithms to merge the filesystems, depending on the file type:
* For `i18n` and `data` files, Hugo merges deeply using the translation id and data key inside the files.
* For `static`, `layouts` (templates) and `archetypes` files, these are merged on file level. So the left-most file will be chosen.
The name used in the `theme` definition above must match a folder in `/your-site/themes`, e.g. `/your-site/themes/my-shortcodes`. There are plans to improve on this and get a URL scheme so this can be resolved automatically.
Also note that a component that is part of a theme can have its own configuration file, e.g. `config.toml`. There are currently some restrictions to what a theme component can configure:
* `params` (global and per language)
* `menu` (global and per language)
* `outputformats` and `mediatypes`
The same rules apply here: The left-most param/menu etc. with the same ID will win. There are some hidden and experimental namespace support in the above, which we will work to improve in the future, but theme authors are encouraged to create their own namespaces to avoid naming conflicts.
A final note: Themes/components can also have a `theme` definition in their `config.toml` and similar, which is the "inheritance" part of this commit's title. This is currently not supported by the Hugo theme site. We will have to wait for some "auto dependency" feature to be implemented for that to happen, but this can be a powerful feature if you want to create your own theme-variant based on others.
Fixes #4460
Fixes #4450
86 files changed, 2820 insertions, 1914 deletions
diff --git a/Gopkg.lock b/Gopkg.lock index 86c9e7c8f..51fb96c52 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -298,8 +298,8 @@ ".", "mem" ] - revision = "63644898a8da0bc22138abf860edaf5277b6102e" - version = "v1.1.0" + revision = "787d034dfe70e44075ccc060d346146ef53270ad" + version = "v1.1.1" [[projects]] name = "github.com/spf13/cast" diff --git a/commands/benchmark.go b/commands/benchmark.go index 3938acf1b..b0a12db7f 100644 --- a/commands/benchmark.go +++ b/commands/benchmark.go @@ -56,7 +56,7 @@ func (c *benchmarkCmd) benchmark(cmd *cobra.Command, args []string) error { return nil } - comm, err := initializeConfig(false, &c.hugoBuilderCommon, c, cfgInit) + comm, err := initializeConfig(true, false, &c.hugoBuilderCommon, c, cfgInit) if err != nil { return err } diff --git a/commands/commandeer.go b/commands/commandeer.go index d43b7c9f1..d5d2740bf 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -34,7 +34,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" - src "github.com/gohugoio/hugo/source" + "github.com/gohugoio/hugo/langs" ) type commandeer struct { @@ -45,11 +45,8 @@ type commandeer struct { h *hugoBuilderCommon ftch flagsToConfigHandler - pathSpec *helpers.PathSpec visitedURLs *types.EvictingStringQueue - staticDirsConfig []*src.Dirs - // We watch these for changes. configFiles []string @@ -63,7 +60,7 @@ type commandeer struct { serverPorts []int languagesConfigured bool - languages helpers.Languages + languages langs.Languages configured bool } @@ -75,31 +72,13 @@ func (c *commandeer) Set(key string, value interface{}) { c.Cfg.Set(key, value) } -// PathSpec lazily creates a new PathSpec, as all the paths must -// be configured before it is created. -func (c *commandeer) PathSpec() *helpers.PathSpec { - c.configured = true - return c.pathSpec -} - func (c *commandeer) initFs(fs *hugofs.Fs) error { c.DepsCfg.Fs = fs - ps, err := helpers.NewPathSpec(fs, c.Cfg) - if err != nil { - return err - } - c.pathSpec = ps - - dirsConfig, err := c.createStaticDirsConfig() - if err != nil { - return err - } - c.staticDirsConfig = dirsConfig return nil } -func newCommandeer(running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) { +func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) { var rebuildDebouncer func(f func()) if running { @@ -117,10 +96,10 @@ func newCommandeer(running bool, h *hugoBuilderCommon, f flagsToConfigHandler, d debounce: rebuildDebouncer, } - return c, c.loadConfig(running) + return c, c.loadConfig(mustHaveConfigFile, running) } -func (c *commandeer) loadConfig(running bool) error { +func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error { if c.DepsCfg == nil { c.DepsCfg = &deps.DepsCfg{} @@ -168,12 +147,18 @@ func (c *commandeer) loadConfig(running bool) error { doWithConfig) if err != nil { - return err + if mustHaveConfigFile { + return err + } + if err != hugolib.ErrNoConfigFile { + return err + } + } c.configFiles = configFiles - if l, ok := c.Cfg.Get("languagesSorted").(helpers.Languages); ok { + if l, ok := c.Cfg.Get("languagesSorted").(langs.Languages); ok { c.languagesConfigured = true c.languages = l } @@ -209,6 +194,15 @@ func (c *commandeer) loadConfig(running bool) error { } err = c.initFs(fs) + if err != nil { + return + } + + var h *hugolib.HugoSites + + h, err = hugolib.NewHugoSites(*c.DepsCfg) + c.hugo = h + }) if err != nil { @@ -232,7 +226,7 @@ func (c *commandeer) loadConfig(running bool) error { cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed()) - themeDir := c.PathSpec().GetThemeDir() + themeDir := c.hugo.PathSpec.GetFirstThemeDir() if themeDir != "" { if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) { return newSystemError("Unable to find theme Directory:", themeDir) diff --git a/commands/commands.go b/commands/commands.go index 8ba28e10d..74bc709cc 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -148,7 +148,7 @@ Complete documentation is available at http://gohugo.io/.`, return nil } - c, err := initializeConfig(cc.buildWatch, &cc.hugoBuilderCommon, cc, cfgInit) + c, err := initializeConfig(true, cc.buildWatch, &cc.hugoBuilderCommon, cc, cfgInit) if err != nil { return err } diff --git a/commands/commands_test.go b/commands/commands_test.go index 907f003c0..d576b4428 100644 --- a/commands/commands_test.go +++ b/commands/commands_test.go @@ -237,6 +237,11 @@ List: {{ .Title }} `) + writeFile(t, filepath.Join(d, "static", "my.txt"), ` +MyMy + +`) + return d, nil } diff --git a/commands/config.go b/commands/config.go index 951b57540..33a61733d 100644 --- a/commands/config.go +++ b/commands/config.go @@ -44,7 +44,7 @@ func newConfigCmd() *configCmd { } func (c *configCmd) printConfig(cmd *cobra.Command, args []string) error { - cfg, err := initializeConfig(false, &c.hugoBuilderCommon, c, nil) + cfg, err := initializeConfig(true, false, &c.hugoBuilderCommon, c, nil) if err != nil { return err diff --git a/commands/convert.go b/commands/convert.go index fb70a148d..8de155e9b 100644 --- a/commands/convert.go +++ b/commands/convert.go @@ -96,7 +96,7 @@ func (cc *convertCmd) convertContents(mark rune) error { return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path") } - c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, nil) + c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, nil) if err != nil { return err } diff --git a/commands/hugo.go b/commands/hugo.go index 8f7860f76..c4fee122d 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -23,6 +23,8 @@ import ( "sync/atomic" "syscall" + "github.com/gohugoio/hugo/hugolib/filesystems" + "golang.org/x/sync/errgroup" "log" @@ -32,8 +34,6 @@ import ( "strings" "time" - src "github.com/gohugoio/hugo/source" - "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/parser" @@ -103,12 +103,12 @@ func Execute(args []string) Response { } // InitializeConfig initializes a config file with sensible default configuration flags. -func initializeConfig(running bool, +func initializeConfig(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error) (*commandeer, error) { - c, err := newCommandeer(running, h, f, doWithCommandeer) + c, err := newCommandeer(mustHaveConfigFile, running, h, f, doWithCommandeer) if err != nil { return nil, err } @@ -280,6 +280,7 @@ func (c *commandeer) fullBuild() error { return fmt.Errorf("Error copying static files: %s", err) } langCount = cnt + langCount = cnt return nil } buildSitesFunc := func() error { @@ -344,7 +345,7 @@ func (c *commandeer) build() error { if err != nil { return err } - c.Logger.FEEDBACK.Println("Watching for changes in", c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir"))) + c.Logger.FEEDBACK.Println("Watching for changes in", c.hugo.PathSpec.AbsPathify(c.Cfg.GetString("contentDir"))) c.Logger.FEEDBACK.Println("Press Ctrl+C to stop") watcher, err := c.newWatcher(watchDirs...) utils.CheckErr(c.Logger, err) @@ -380,49 +381,30 @@ func (c *commandeer) copyStatic() (map[string]uint64, error) { return c.doWithPublishDirs(c.copyStaticTo) } -func (c *commandeer) createStaticDirsConfig() ([]*src.Dirs, error) { - var dirsConfig []*src.Dirs - - if !c.languages.IsMultihost() { - dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger) - if err != nil { - return nil, err - } - dirsConfig = append(dirsConfig, dirs) - } else { - for _, l := range c.languages { - dirs, err := src.NewDirs(c.Fs, l, c.DepsCfg.Logger) - if err != nil { - return nil, err - } - dirsConfig = append(dirsConfig, dirs) - } - } - - return dirsConfig, nil - -} - -func (c *commandeer) doWithPublishDirs(f func(dirs *src.Dirs, publishDir string) (uint64, error)) (map[string]uint64, error) { +func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesystem) (uint64, error)) (map[string]uint64, error) { langCount := make(map[string]uint64) - for _, dirs := range c.staticDirsConfig { + staticFilesystems := c.hugo.BaseFs.SourceFilesystems.Static - cnt, err := f(dirs, c.pathSpec.PublishDir) + if len(staticFilesystems) == 0 { + c.Logger.WARN.Println("No static directories found to sync") + return langCount, nil + } + + for lang, fs := range staticFilesystems { + cnt, err := f(fs) if err != nil { return langCount, err } - - if dirs.Language == nil { + if lang == "" { // Not multihost for _, l := range c.languages { langCount[l.Lang] = cnt } } else { - langCount[dirs.Language.Lang] = cnt + langCount[lang] = cnt } - } return langCount, nil @@ -443,29 +425,18 @@ func (fs *countingStatFs) Stat(name string) (os.FileInfo, error) { return f, err } -func (c *commandeer) copyStaticTo(dirs *src.Dirs, publishDir string) (uint64, error) { - +func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint64, error) { + publishDir := c.hugo.PathSpec.PublishDir // If root, remove the second '/' if publishDir == "//" { publishDir = helpers.FilePathSeparator } - if dirs.Language != nil { - // Multihost setup. - publishDir = filepath.Join(publishDir, dirs.Language.Lang) + if sourceFs.PublishFolder != "" { + publishDir = filepath.Join(publishDir, sourceFs.PublishFolder) } - staticSourceFs, err := dirs.CreateStaticFs() - if err != nil { - return 0, err - } - - if staticSourceFs == nil { - c.Logger.WARN.Println("No static directories found to sync") - return 0, nil - } - - fs := &countingStatFs{Fs: staticSourceFs} + fs := &countingStatFs{Fs: sourceFs.Fs} syncer := fsync.NewSyncer() syncer.NoTimes = c.Cfg.GetBool("noTimes") @@ -485,6 +456,8 @@ func (c *commandeer) copyStaticTo(dirs *src.Dirs, publishDir string) (uint64, er } c.Logger.INFO.Println("syncing static files to", publishDir) + var err error + // because we are using a baseFs (to get the union right). // set sync src to root err = syncer.Sync(publishDir, helpers.FilePathSeparator) @@ -514,41 +487,10 @@ func (c *commandeer) getDirList() ([]string, error) { var seen = make(map[string]bool) var nested []string - dataDir := c.PathSpec().AbsPathify(c.Cfg.GetString("dataDir")) - i18nDir := c.PathSpec().AbsPathify(c.Cfg.GetString("i18nDir")) - staticSyncer, err := newStaticSyncer(c) - if err != nil { - return nil, err - } - - layoutDir := c.PathSpec().GetLayoutDirPath() - staticDirs := staticSyncer.d.AbsStaticDirs - newWalker := func(allowSymbolicDirs bool) func(path string, fi os.FileInfo, err error) error { return func(path string, fi os.FileInfo, err error) error { if err != nil { - if path == dataDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip dataDir:", err) - return nil - } - - if path == i18nDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip i18nDir:", err) - return nil - } - - if path == layoutDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip layoutDir:", err) - return nil - } - if os.IsNotExist(err) { - for _, staticDir := range staticDirs { - if path == staticDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip staticDir:", err) - } - } - // Ignore. return nil } @@ -605,23 +547,28 @@ func (c *commandeer) getDirList() ([]string, error) { regularWalker := newWalker(false) // SymbolicWalk will log anny ERRORs - _ = helpers.SymbolicWalk(c.Fs.Source, dataDir, regularWalker) - _ = helpers.SymbolicWalk(c.Fs.Source, i18nDir, regularWalker) - _ = helpers.SymbolicWalk(c.Fs.Source, layoutDir, regularWalker) - - for _, contentDir := range c.PathSpec().ContentDirs() { + // Also note that the Dirnames fetched below will contain any relevant theme + // directories. + for _, contentDir := range c.hugo.PathSpec.BaseFs.AbsContentDirs { _ = helpers.SymbolicWalk(c.Fs.Source, contentDir.Value, symLinkWalker) } - for _, staticDir := range staticDirs { + for _, staticDir := range c.hugo.PathSpec.BaseFs.Data.Dirnames { _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) } - if c.PathSpec().ThemeSet() { - themesDir := c.PathSpec().GetThemeDir() - _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "layouts"), regularWalker) - _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "i18n"), regularWalker) - _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "data"), regularWalker) + for _, staticDir := range c.hugo.PathSpec.BaseFs.I18n.Dirnames { + _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) + } + + for _, staticDir := range c.hugo.PathSpec.BaseFs.Layouts.Dirnames { + _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) + } + + for _, staticFilesystem := range c.hugo.PathSpec.BaseFs.Static { + for _, staticDir := range staticFilesystem.Dirnames { + _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) + } } if len(nested) > 0 { @@ -648,9 +595,6 @@ func (c *commandeer) getDirList() ([]string, error) { func (c *commandeer) recreateAndBuildSites(watching bool) (err error) { defer c.timeTrack(time.Now(), "Total") - if err := c.initSites(); err != nil { - return err - } if !c.h.quiet { c.Logger.FEEDBACK.Println("Started building sites ...") } @@ -658,56 +602,30 @@ func (c *commandeer) recreateAndBuildSites(watching bool) (err error) { } func (c *commandeer) resetAndBuildSites() (err error) { - if err = c.initSites(); err != nil { - return - } if !c.h.quiet { c.Logger.FEEDBACK.Println("Started building sites ...") } return c.hugo.Build(hugolib.BuildCfg{ResetState: true}) } -func (c *commandeer) initSites() error { - if c.hugo != nil { - c.hugo.Cfg = c.Cfg - return nil - } - - h, err := hugolib.NewHugoSites(*c.DepsCfg) - - if err != nil { - return err - } - - c.hugo = h - - return nil -} - func (c *commandeer) buildSites() (err error) { - if err := c.initSites(); err != nil { - return err - } return c.hugo.Build(hugolib.BuildCfg{}) } func (c *commandeer) rebuildSites(events []fsnotify.Event) error { defer c.timeTrack(time.Now(), "Total") - if err := c.initSites(); err != nil { - return err - } visited := c.visitedURLs.PeekAllSet() doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") if doLiveReload && !c.Cfg.GetBool("disableFastRender") { // Make sure we always render the home pages for _, l := range c.languages { - langPath := c.PathSpec().GetLangSubDir(l.Lang) + langPath := c.hugo.PathSpec.GetLangSubDir(l.Lang) if langPath != "" { langPath = langPath + "/" } - home := c.pathSpec.PrependBasePath("/" + langPath) + home := c.hugo.PathSpec.PrependBasePath("/" + langPath) visited[home] = true } @@ -716,7 +634,7 @@ func (c *commandeer) rebuildSites(events []fsnotify.Event) error { } func (c *commandeer) fullRebuild() { - if err := c.loadConfig(true); err != nil { + if err := c.loadConfig(true, true); err != nil { jww.ERROR.Println("Failed to reload config:", err) } else if err := c.recreateAndBuildSites(true); err != nil { jww.ERROR.Println(err) @@ -906,7 +824,8 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) { // force refresh when more than one file if len(staticEvents) > 0 { for _, ev := range staticEvents { - path := staticSyncer.d.MakeStaticPathRelative(ev.Name) + + path := c.hugo.BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name) livereload.RefreshPath(path) } @@ -975,32 +894,36 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string { } // isThemeVsHugoVersionMismatch returns whether the current Hugo version is -// less than the theme's min_version. +// less than any of the themes' min_version. func (c *commandeer) isThemeVsHugoVersionMismatch(fs afero.Fs) (mismatch bool, requiredMinVersion string) { - if !c.PathSpec().ThemeSet() { + if !c.hugo.PathSpec.ThemeSet() { return } - themeDir := c.PathSpec().GetThemeDir() + for _, absThemeDir := range c.hugo.BaseFs.AbsThemeDirs { - path := filepath.Join(themeDir, "theme.toml") + path := filepath.Join(absThemeDir, "theme.toml") - exists, err := helpers.Exists(path, fs) + exists, err := helpers.Exists(path, fs) - if err != nil || !exists { - return - } + if err != nil || !exists { + continue + } - b, err := afero.ReadFile(fs, path) + b, err := afero.ReadFile(fs, path) - tomlMeta, err := parser.HandleTOMLMetaData(b) + tomlMeta, err := parser.HandleTOMLMetaData(b) - if err != nil { - return - } + if err != nil { + continue + } + + if minVersion, ok := tomlMeta["min_version"]; ok { + if helpers.CompareVersion(minVersion) > 0 { + return true, fmt.Sprint(minVersion) + } + } - if minVersion, ok := tomlMeta["min_version"]; ok { - return helpers.CompareVersion(minVersion) > 0, fmt.Sprint(minVersion) } return diff --git a/commands/list.go b/commands/list.go index 57a92082c..9922e957d 100644 --- a/commands/list.go +++ b/commands/list.go @@ -50,7 +50,7 @@ List requires a subcommand, e.g. ` + "`hugo list drafts`.", c.Set("buildDrafts", true) return nil } - c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, cfgInit) + c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit) if err != nil { return err } @@ -86,7 +86,7 @@ posted in the future.`, c.Set("buildFuture", true) return nil } - c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, cfgInit) + c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit) if err != nil { return err } @@ -122,7 +122,7 @@ expired.`, c.Set("buildExpired", true) return nil } - c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, cfgInit) + c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit) if err != nil { return err } diff --git a/commands/new.go b/commands/new.go index c088dca9b..27d079b0d 100644 --- a/commands/new.go +++ b/commands/new.go @@ -71,7 +71,7 @@ func (n *newCmd) newContent(cmd *cobra.Command, args []string) error { return nil } - c, err := initializeConfig(false, &n.hugoBuilderCommon, n, cfgInit) + c, err := initializeConfig(true, false, &n.hugoBuilderCommon, n, cfgInit) if err != nil { return err @@ -104,9 +104,6 @@ func (n *newCmd) newContent(cmd *cobra.Command, args []string) error { return hugolib.NewSite(*cfg) } var s *hugolib.Site - if err := c.initSites(); err != nil { - return nil, err - } if err := c.hugo.Build(hugolib.BuildCfg{SkipRender: true}); err != nil { return nil, err diff --git a/commands/new_theme.go b/commands/new_theme.go index 3b00cb1df..9464e1968 100644 --- a/commands/new_theme.go +++ b/commands/new_theme.go @@ -54,7 +54,7 @@ as you see fit.`, } func (n *newThemeCmd) newTheme(cmd *cobra.Command, args []string) error { - c, err := initializeConfig(false, &n.hugoBuilderCommon, n, nil) + c, err := initializeConfig(false, false, &n.hugoBuilderCommon, n, nil) if err != nil { return err @@ -64,7 +64,7 @@ func (n *newThemeCmd) newTheme(cmd *cobra.Command, args []string) error { return newUserError("theme name needs to be provided") } - createpath := c.PathSpec().AbsPathify(filepath.Join(c.Cfg.GetString("themesDir"), args[0])) + createpath := c.hugo.PathSpec.AbsPathify(filepath.Join(c.Cfg.GetString("themesDir"), args[0])) jww.INFO.Println("creating theme at", createpath) cfg := c.DepsCfg @@ -140,7 +140,7 @@ description = "" homepage = "http://example.com/" tags = [] features = [] -min_version = "0.38" +min_version = "0.41" [author] name = "" diff --git a/commands/server.go b/commands/server.go index c05180de9..8089b0ade 100644 --- a/commands/server.go +++ b/commands/server.go @@ -226,7 +226,7 @@ func (s *serverCmd) server(cmd *cobra.Command, args []string) error { jww.ERROR.Println("memstats error:", err) } - c, err := initializeConfig(true, &s.hugoBuilderCommon, s, cfgInit) + c, err := initializeConfig(true, true, &s.hugoBuilderCommon, s, cfgInit) if err != nil { return err } @@ -288,7 +288,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro publishDir = filepath.Join(publishDir, root) } - absPublishDir := f.c.PathSpec().AbsPathify(publishDir) + absPublishDir := f.c.hugo.PathSpec.AbsPathify(publishDir) if i == 0 { if f.s.renderToDisk { diff --git a/commands/static_syncer.go b/commands/static_syncer.go index a04904f95..1e73e7fc2 100644 --- a/commands/static_syncer.go +++ b/commands/static_syncer.go @@ -17,53 +17,43 @@ import ( "os" "path/filepath" + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/helpers" - src "github.com/gohugoio/hugo/source" "github.com/spf13/fsync" ) type staticSyncer struct { c *commandeer - d *src.Dirs } func newStaticSyncer(c *commandeer) (*staticSyncer, error) { - dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger) - if err != nil { - return nil, err - } - - return &staticSyncer{c: c, d: dirs}, nil + return &staticSyncer{c: c}, nil } -func (s *staticSyncer) isStatic(path string) bool { - return s.d.IsStatic(path) +func (s *staticSyncer) isStatic(filename string) bool { + return s.c.hugo.BaseFs.SourceFilesystems.IsStatic(filename) } func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error { c := s.c - syncFn := func(dirs *src.Dirs, publishDir string) (uint64, error) { - staticSourceFs, err := dirs.CreateStaticFs() - if err != nil { - return 0, err - } - - if dirs.Language != nil { - // Multihost setup - publishDir = filepath.Join(publishDir, dirs.Language.Lang) + syncFn := func(sourceFs *filesystems.SourceFilesystem) (uint64, error) { + publishDir := c.hugo.PathSpec.PublishDir + // If root, remove the second '/' + if publishDir == "//" { + publishDir = helpers.FilePathSeparator } - if staticSourceFs == nil { - c.Logger.WARN.Println("No static directories found to sync") - return 0, nil + if sourceFs.PublishFolder != "" { + publishDir = filepath.Join(publishDir, sourceFs.PublishFolder) } syncer := fsync.NewSyncer() syncer.NoTimes = c.Cfg.GetBool("noTimes") syncer.NoChmod = c.Cfg.GetBool("noChmod") - syncer.SrcFs = staticSourceFs + syncer.SrcFs = sourceFs.Fs syncer.DestFs = c.Fs.Destination // prevent spamming the log on changes @@ -88,8 +78,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error { fromPath := ev.Name - // If we are here we already know the event took place in a static dir - relPath := dirs.MakeStaticPathRelative(fromPath) + relPath := sourceFs.MakePathRelative(fromPath) if relPath == "" { // Not member of this virtual host. continue @@ -105,7 +94,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error { // the source of that static file. In this case Hugo will incorrectly remove that file // from the published directory. if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove { - if _, err := staticSourceFs.Stat(relPath); os.IsNotExist(err) { + if _, err := sourceFs.Fs.Stat(relPath); os.IsNotExist(err) { // If file doesn't exist in any static dir, remove it toRemove := filepath.Join(publishDir, relPath) diff --git a/common/loggers/loggers.go b/common/loggers/loggers.go new file mode 100644 index 000000000..2f7f36b34 --- /dev/null +++ b/common/loggers/loggers.go @@ -0,0 +1,37 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loggers + +import ( + "io/ioutil" + "log" + "os" + + jww "github.com/spf13/jwalterweatherman" +) + +// NewDebugLogger is a convenience function to create a debug logger. +func NewDebugLogger() *jww.Notepad { + return jww.NewNotepad(jww.LevelDebug, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) +} + +// NewWarningLogger is a convenience function to create a warning logger. +func NewWarningLogger() *jww.Notepad { + return jww.NewNotepad(jww.LevelWarn, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) +} + +// NewErrorLogger is a convenience function to create an error logger. +func NewErrorLogger() *jww.Notepad { + return jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) +} diff --git a/common/maps/maps.go b/common/maps/maps.go new file mode 100644 index 000000000..a114b557c --- /dev/null +++ b/common/maps/maps.go @@ -0,0 +1,44 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maps + +import ( + "strings" + + "github.com/spf13/cast" +) + +// ToLower makes all the keys in the given map lower cased and will do so +// recursively. +// Notes: +// * This will modify the map given. +// * Any nested map[interface{}]interface{} will be converted to map[string]interface{}. +func ToLower(m map[string]interface{}) { + for k, v := range m { + switch v.(type) { + case map[interface{}]interface{}: + v = cast.ToStringMap(v) + ToLower(v.(map[string]interface{})) + case map[string]interface{}: + ToLower(v.(map[string]interface{})) + } + + lKey := strings.ToLower(k) + if k != lKey { + delete(m, k) + m[lKey] = v + } + + } +} diff --git a/common/maps/maps_test.go b/common/maps/maps_test.go new file mode 100644 index 000000000..37add5dc5 --- /dev/null +++ b/common/maps/maps_test.go @@ -0,0 +1,72 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maps + +import ( + "reflect" + "testing" +) + +func TestToLower(t *testing.T) { + + tests := []struct { + input map[string]interface{} + expected map[string]interface{} + }{ + { + map[string]interface{}{ + "abC": 32, + }, + map[string]interface{}{ + "abc": 32, + }, + }, + { + map[string]interface{}{ + "abC": 32, + "deF": map[interface{}]interface{}{ + 23: "A value", + 24: map[string]interface{}{ + "AbCDe": "A value", + "eFgHi": "Another value", + }, + }, + "gHi": map[string]interface{}{ + "J": 25, + }, + }, + map[string]interface{}{ + "abc": 32, + "def": map[string]interface{}{ + "23": "A value", + "24": map[string]interface{}{ + "abcde": "A value", + "efghi": "Another value", + }, + }, + "ghi": map[string]interface{}{ + "j": 25, + }, + }, + }, + } + + for i, test := range tests { + // ToLower modifies input. + ToLower(test.input) + if !reflect.DeepEqual(test.expected, test.input) { + t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, test.input) + } + } +} diff --git a/config/configProvider.go b/config/configProvider.go index 335294d73..a7dc18960 100644 --- a/config/configProvider.go +++ b/config/configProvider.go @@ -16,6 +16,8 @@ package config import ( "strings" + "github.com/spf13/cast" + "github.com/spf13/viper" ) @@ -40,5 +42,16 @@ func FromConfigString(config, configType string) (Provider, error) { return nil, err } return v, nil +} +// GetStringSlicePreserveString returns a string slice from the given config and key. +// It differs from the GetStringSlice method in that if the config value is a string, +// we do not attempt to split it into fields. +func GetStringSlicePreserveString(cfg Provider, key string) []string { + sd := cfg.Get(key) + if sds, ok := sd.(string); ok { + return []string{sds} + } else { + return cast.ToStringSlice(sd) + } } diff --git a/config/configProvider_test.go b/config/configProvider_test.go new file mode 100644 index 000000000..7e9c2223b --- /dev/null +++ b/config/configProvider_test.go @@ -0,0 +1,36 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestGetStringSlicePreserveString(t *testing.T) { + assert := require.New(t) + cfg := viper.New() + + s := "This is a string" + sSlice := []string{"This", "is", "a", "slice"} + + cfg.Set("s1", s) + cfg.Set("s2", sSlice) + + assert.Equal([]string{s}, GetStringSlicePreserveString(cfg, "s1")) + assert.Equal(sSlice, GetStringSlicePreserveString(cfg, "s2")) + assert.Nil(GetStringSlicePreserveString(cfg, "s3")) +} diff --git a/create/content.go b/create/content.go index 29fe47394..6d022282e 100644 --- a/create/content.go +++ b/create/content.go @@ -16,6 +16,7 @@ package create import ( "bytes" + "fmt" "os" "os/exec" "path/filepath" @@ -31,6 +32,7 @@ func NewContent( ps *helpers.PathSpec, siteFactory func(filename string, siteUsed bool) (*hugolib.Site, error), kind, targetPath string) error { ext := helpers.Ext(targetPath) + fs := ps.BaseFs.SourceFilesystems.Archetypes.Fs jww.INFO.Printf("attempting to create %q of %q of ext %q", targetPath, kind, ext) @@ -40,9 +42,9 @@ func NewContent( siteUsed := false if archetypeFilename != "" { - f, err := ps.Fs.Source.Open(archetypeFilename) + f, err := fs.Open(archetypeFilename) if err != nil { - return err + return fmt.Errorf("failed to open archetype file: %s", err) } defer f.Close() @@ -71,7 +73,7 @@ func NewContent( targetDir := filepath.Dir(targetPath) if targetDir != "" && targetDir != "." { - exists, _ = helpers.Exists(targetDir, ps.Fs.Source) + exists, _ = helpers.Exists(targetDir, fs) } if exists { @@ -101,42 +103,27 @@ func NewContent( return nil } -// FindArchetype takes a given kind/archetype of content and returns an output -// path for that archetype. If no archetype is found, an empty string is -// returned. +// FindArchetype takes a given kind/archetype of content and returns the path +// to the archetype in the archetype filesystem, blank if none found. func findArchetype(ps *helpers.PathSpec, kind, ext string) (outpath string) { - search := []string{ps.AbsPathify(ps.Cfg.GetString("archetypeDir"))} + fs := ps.BaseFs.Archetypes.Fs - if ps.Cfg.GetString("theme") != "" { - themeDir := filepath.Join(ps.AbsPathify(ps.Cfg.GetString("themesDir")+"/"+ps.Cfg.GetString("theme")), "/archetypes/") - if _, err := ps.Fs.Source.Stat(themeDir); os.IsNotExist(err) { - jww.ERROR.Printf("Unable to find archetypes directory for theme %q at %q", ps.Cfg.GetString("theme"), themeDir) + // If the new content isn't in a subdirectory, kind == "". + // Therefore it should be excluded otherwise `is a directory` + // error will occur. github.com/gohugoio/hugo/issues/411 + var pathsToCheck = []string{"default"} + + if ext != "" { + if kind != "" { + pathsToCheck = append([]string{kind + ext, "default" + ext}, pathsToCheck...) } else { - search = append(search, themeDir) + pathsToCheck = append([]string{"default" + ext}, pathsToCheck...) } } - for _, x := range search { - // If the new content isn't in a subdirectory, kind == "". - // Therefore it should be excluded otherwise `is a directory` - // error will occur. github.com/gohugoio/hugo/issues/411 - var pathsToCheck = []string{"default"} - - if ext != "" { - if kind != "" { - pathsToCheck = append([]string{kind + ext, "default" + ext}, pathsToCheck...) - } else { - pathsToCheck = append([]string{"default" + ext}, pathsToCheck...) - } - } - - for _, p := range pathsToCheck { - curpath := filepath.Join(x, p) - jww.DEBUG.Println("checking", curpath, "for archetypes") - if exists, _ := helpers.Exists(curpath, ps.Fs.Source); exists { - jww.INFO.Println("curpath: " + curpath) - return curpath - } + for _, p := range pathsToCheck { + if exists, _ := helpers.Exists(p, fs); exists { + return p } } diff --git a/create/content_template_handler.go b/create/content_template_handler.go index 17e52cae0..37eed52cf 100644 --- a/create/content_template_handler.go +++ b/create/content_template_handler.go @@ -89,10 +89,11 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile ) ps, err := helpers.NewPathSpec(s.Deps.Fs, s.Deps.Cfg) - sp := source.NewSourceSpec(ps, ps.Fs.Source) if err != nil { return nil, err } + sp := source.NewSourceSpec(ps, ps.Fs.Source) + f := sp.NewFileInfo("", targetPath, false, nil) name := f.TranslationBaseName() @@ -115,9 +116,9 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile // TODO(bep) archetype revive the issue about wrong tpl funcs arg order archetypeTemplate = []byte(ArchetypeTemplateTemplate) } else { - archetypeTemplate, err = afero.ReadFile(s.Fs.Source, archetypeFilename) + archetypeTemplate, err = afero.ReadFile(s.BaseFs.Archetypes.Fs, archetypeFilename) if err != nil { - return nil, fmt.Errorf("Failed to read archetype file %q: %s", archetypeFilename, err) + return nil, fmt.Errorf("failed to read archetype file %s", err) } } diff --git a/create/content_test.go b/create/content_test.go index 62d5ed1da..e9d46becf 100644 --- a/create/content_test.go +++ b/create/content_test.go @@ -58,17 +58,15 @@ func TestNewContent(t *testing.T) { for _, c := range cases { cfg, fs := newTestCfg() - ps, err := helpers.NewPathSpec(fs, cfg) - require.NoError(t, err) + require.NoError(t, initFs(fs)) h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) require.NoError(t, err) - require.NoError(t, initFs(fs)) siteFactory := func(filename string, siteUsed bool) (*hugolib.Site, error) { return h.Sites[0], nil } - require.NoError(t, create.NewContent(ps, siteFactory, c.kind, c.path)) + require.NoError(t, create.NewContent(h.PathSpec, siteFactory, c.kind, c.path)) fname := filepath.Join("content", filepath.FromSlash(c.path)) content := readFileFromFs(t, fs.Source, fname) @@ -89,6 +87,7 @@ func initViper(v *viper.Viper) { v.Set("layoutDir", "layouts") v.Set("i18nDir", "i18n") v.Set("theme", "sample") + v.Set("archetypeDir", "archetypes") } func initFs(fs *hugofs.Fs) error { @@ -187,6 +186,12 @@ func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string { func newTestCfg() (*viper.Viper, *hugofs.Fs) { v := viper.New() + v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") + fs := hugofs.NewMem(v) v.SetFs(fs.Source) diff --git a/deps/deps.go b/deps/deps.go index 733de03b3..d233025d3 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -9,6 +9,7 @@ import ( "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/metrics" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/source" @@ -47,7 +48,7 @@ type Deps struct { // The translation func to use Translate func(translationID string, args ...interface{}) string `json:"-"` - Language *helpers.Language + Language *langs.Language // All the output formats available for the current site. OutputFormatsConfig output.Formats @@ -166,10 +167,10 @@ func New(cfg DepsCfg) (*Deps, error) { // ForLanguage creates a copy of the Deps with the language dependent // parts switched out. -func (d Deps) ForLanguage(l *helpers.Language) (*Deps, error) { +func (d Deps) ForLanguage(l *langs.Language) (*Deps, error) { var err error - d.PathSpec, err = helpers.NewPathSpec(d.Fs, l) + d.PathSpec, err = helpers.NewPathSpecWithBaseBaseFsProvided(d.Fs, l, d.BaseFs) if err != nil { return nil, err } @@ -206,7 +207,7 @@ type DepsCfg struct { Fs *hugofs.Fs // The language to use. - Language *helpers.Language + Language *langs.Language // The configuration to use. Cfg config.Provider diff --git a/helpers/content.go b/helpers/content.go index 1c0a7b7e9..55d8ce202 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -25,6 +25,8 @@ import ( "unicode" "unicode/utf8" + "github.com/gohugoio/hugo/common/maps" + "github.com/chaseadamsio/goorgeous" bp "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/config" @@ -134,7 +136,7 @@ func newBlackfriday(config map[string]interface{}) *BlackFriday { "taskLists": true, } - ToLowerMap(defaultParam) + maps.ToLower(defaultParam) siteConfig := make(map[string]interface{}) diff --git a/helpers/general.go b/helpers/general.go index 5b46520e5..b442b1eb4 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -20,18 +20,20 @@ import ( "fmt" "io" "net" + "os" "path/filepath" "strings" "sync" "unicode" "unicode/utf8" + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/afero" "github.com/jdkato/prose/transform" bp "github.com/gohugoio/hugo/bufferpool" - "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" "github.com/spf13/pflag" ) @@ -129,30 +131,6 @@ func ReaderToBytes(lines io.Reader) []byte { return bc } -// ToLowerMap makes all the keys in the given map lower cased and will do so -// recursively. -// Notes: -// * This will modify the map given. -// * Any nested map[interface{}]interface{} will be converted to map[string]interface{}. -func ToLowerMap(m map[string]interface{}) { - for k, v := range m { - switch v.(type) { - case map[interface{}]interface{}: - v = cast.ToStringMap(v) - ToLowerMap(v.(map[string]interface{})) - case map[string]interface{}: - ToLowerMap(v.(map[string]interface{})) - } - - lKey := strings.ToLower(k) - if k != lKey { - delete(m, k) - m[lKey] = v - } - - } -} - // ReaderToString is the same as ReaderToBytes, but returns a string. func ReaderToString(lines io.Reader) string { if lines == nil { @@ -255,11 +233,6 @@ func compareStringSlices(a, b []string) bool { return true } -// ThemeSet checks whether a theme is in use or not. -func (p *PathSpec) ThemeSet() bool { - return p.theme != "" -} - // LogPrinter is the common interface of the JWWs loggers. type LogPrinter interface { // Println is the only common method that works in all of JWWs loggers. @@ -477,3 +450,24 @@ func DiffStringSlices(slice1 []string, slice2 []string) []string { func DiffStrings(s1, s2 string) []string { return DiffStringSlices(strings.Fields(s1), strings.Fields(s2)) } + +// PrintFs prints the given filesystem to the given writer starting from the given path. +// This is useful for debugging. +func PrintFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return + } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + if info != nil && !info.IsDir() { + s := path + if lang, ok := info.(hugofs.LanguageAnnouncer); ok { + s = s + "\tLANG: " + lang.Lang() + } + if fp, ok := info.(hugofs.FilePather); ok { + s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir() + } + fmt.Fprintln(w, " ", s) + } + return nil + }) +} diff --git a/helpers/general_test.go b/helpers/general_test.go index 16df69d24..08fe4890e 100644 --- a/helpers/general_test.go +++ b/helpers/general_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -220,59 +220,6 @@ func TestFindAvailablePort(t *testing.T) { assert.True(t, addr.Port > 0) } -func TestToLowerMap(t *testing.T) { - - tests := []struct { - input map[string]interface{} - expected map[string]interface{} - }{ - { - map[string]interface{}{ - "abC": 32, - }, - map[string]interface{}{ - "abc": 32, - }, - }, - { - map[string]interface{}{ - "abC": 32, - "deF": map[interface{}]interface{}{ - 23: "A value", - 24: map[string]interface{}{ - "AbCDe": "A value", - "eFgHi": "Another value", - }, - }, - "gHi": map[string]interface{}{ - "J": 25, - }, - }, - map[string]interface{}{ - "abc": 32, - "def": map[string]interface{}{ - "23": "A value", - "24": map[string]interface{}{ - "abcde": "A value", - "efghi": "Another value", - }, - }, - "ghi": map[string]interface{}{ - "j": 25, - }, - }, - }, - } - - for i, test := range tests { - // ToLowerMap modifies input. - ToLowerMap(test.input) - if !reflect.DeepEqual(test.expected, test.input) { - t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, test.input) - } - } -} - func TestFastMD5FromFile(t *testing.T) { fs := afero.NewMemMapFs() diff --git a/helpers/path.go b/helpers/path.go index 7ac9208bf..76f13d653 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -20,6 +20,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" "unicode" @@ -31,9 +32,6 @@ import ( var ( // ErrThemeUndefined is returned when a theme has not be defined by the user. ErrThemeUndefined = errors.New("no theme set") - - // ErrWalkRootTooShort is returned when the root specified for a file walk is shorter than 4 characters. - ErrPathTooShort = errors.New("file path is too short") ) // filepathPathBridge is a bridge for common functionality in filepath vs path @@ -86,7 +84,7 @@ func (p *PathSpec) MakePath(s string) string { // MakePathSanitized creates a Unicode-sanitized string, with the spaces replaced func (p *PathSpec) MakePathSanitized(s string) string { - if p.disablePathToLower { + if p.DisablePathToLower { return p.MakePath(s) } return strings.ToLower(p.MakePath(s)) @@ -129,7 +127,7 @@ func (p *PathSpec) UnicodeSanitize(s string) string { var result string - if p.removePathAccents { + if p.RemovePathAccents { // remove accents - see https://blog.golang.org/normalization t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC) result, _, _ = transform.String(t, string(target)) @@ -151,32 +149,19 @@ func ReplaceExtension(path string, newExt string) string { return f + "." + newExt } -// AbsPathify creates an absolute path if given a relative path. If already -// absolute, the path is just cleaned. -func (p *PathSpec) AbsPathify(inPath string) string { - return AbsPathify(p.workingDir, inPath) -} - -// AbsPathify creates an absolute path if given a working dir and arelative path. -// If already absolute, the path is just cleaned. -func AbsPathify(workingDir, inPath string) string { - if filepath.IsAbs(inPath) { - return filepath.Clean(inPath) +// GetFirstThemeDir gets the root directory of the first theme, if there is one. +// If there is no theme, returns the empty string. +func (p *PathSpec) GetFirstThemeDir() string { + if p.ThemeSet() { + return p.AbsPathify(filepath.Join(p.ThemesDir, p.Themes()[0])) } - return filepath.Join(workingDir, inPath) -} - -// GetLayoutDirPath returns the absolute path to the layout file dir -// for the current Hugo project. -func (p *PathSpec) GetLayoutDirPath() string { - return p.AbsPathify(p.layoutDir) + return "" } -// GetThemeDir gets the root directory of the current theme, if there is one. -// If there is no theme, returns the empty string. -func (p *PathSpec) GetThemeDir() string { +// GetThemesDir gets the absolute root theme dir path. +func (p *PathSpec) GetThemesDir() string { if p.ThemeSet() { - return p.AbsPathify(filepath.Join(p.themesDir, p.theme)) + return p.AbsPathify(p.ThemesDir) } return "" } @@ -185,50 +170,11 @@ func (p *PathSpec) GetThemeDir() string { // If there is no theme, returns the empty string. func (p *PathSpec) GetRelativeThemeDir() string { if p.ThemeSet() { - return strings.TrimPrefix(filepath.Join(p.themesDir, p.theme), FilePathSeparator) + return strings.TrimPrefix(filepath.Join(p.ThemesDir, p.Themes()[0]), FilePathSeparator) } return "" } -// GetThemeStaticDirPath returns the theme's static dir path if theme is set. -// If theme is set and the static dir doesn't exist, an error is returned. -func (p *PathSpec) GetThemeStaticDirPath() (string, error) { - return p.getThemeDirPath("static") -} - -// GetThemeDataDirPath returns the theme's data dir path if theme is set. -// If theme is set and the data dir doesn't exist, an error is returned. -func (p *PathSpec) GetThemeDataDirPath() (string, error) { - return p.getThemeDirPath("data") -} - -// GetThemeI18nDirPath returns the theme's i18n dir path if theme is set. -// If theme is set and the i18n dir doesn't exist, an error is returned. -func (p *PathSpec) GetThemeI18nDirPath() (string, error) { - return p.getThemeDirPath("i18n") -} - -func (p *PathSpec) getThemeDirPath(path string) (string, error) { - if !p.ThemeSet() { - return "", ErrThemeUndefined - } - - themeDir := filepath.Join(p.GetThemeDir(), path) - if _, err := p.Fs.Source.Stat(themeDir); os.IsNotExist(err) { - return "", fmt.Errorf("Unable to find %s directory for theme %s in %s", path, p.theme, themeDir) - } - - return themeDir, nil -} - -// GetThemesDirPath gets the static files directory of the current theme, if there is one. -// Ignores underlying errors. -// TODO(bep) Candidate for deprecation? -func (p *PathSpec) GetThemesDirPath() string { - dir, _ := p.getThemeDirPath("static") - return dir -} - func makePathRelative(inPath string, possibleDirectories ...string) (string, error) { for _, currentPath := range possibleDirectories { @@ -445,8 +391,8 @@ func FindCWD() (string, error) { func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error { // Sanity check - if len(root) < 4 { - return ErrPathTooShort + if root != "" && len(root) < 4 { + return errors.New("Path is too short") } // Handle the root first @@ -464,7 +410,10 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error { return err } - rootContent, err := afero.ReadDir(fs, root) + // Some of Hugo's filesystems represents an ordered root folder, i.e. project first, then theme folders. + // Make sure that order is preserved. afero.Walk will sort the directories down in the file tree, + // but we don't care about that. + rootContent, err := readDir(fs, root, false) if err != nil { return walker(root, nil, err) @@ -480,6 +429,22 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error { } +func readDir(fs afero.Fs, dirname string, doSort bool) ([]os.FileInfo, error) { + f, err := fs.Open(dirname) + if err != nil { + return nil, err + } + list, err := f.Readdir(-1) + f.Close() + if err != nil { + return nil, err + } + if doSort { + sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() }) + } + return list, nil +} + func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) { fileInfo, err := LstatIfPossible(fs, path) realPath := path diff --git a/helpers/path_test.go b/helpers/path_test.go index c2ac19675..2c6cb9f37 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -25,6 +25,8 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/langs" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/assert" @@ -56,11 +58,10 @@ func TestMakePath(t *testing.T) { } for _, test := range tests { - v := viper.New() - v.Set("contentDir", "content") + v := newTestCfg() v.Set("removePathAccents", test.removeAccents) - l := NewDefaultLanguage(v) + l := langs.NewDefaultLanguage(v) p, err := NewPathSpec(hugofs.NewMem(v), l) require.NoError(t, err) @@ -74,8 +75,12 @@ func TestMakePath(t *testing.T) { func TestMakePathSanitized(t *testing.T) { v := viper.New() v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") - l := NewDefaultLanguage(v) + l := langs.NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) tests := []struct { @@ -99,12 +104,11 @@ func TestMakePathSanitized(t *testing.T) { } func TestMakePathSanitizedDisablePathToLower(t *testing.T) { - v := viper.New() + v := newTestCfg() v.Set("disablePathToLower", true) - v.Set("contentDir", "content") - l := NewDefaultLanguage(v) + l := langs.NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) tests := []struct { diff --git a/helpers/pathspec.go b/helpers/pathspec.go index b18408590..847029f44 100644 --- a/helpers/pathspec.go +++ b/helpers/pathspec.go @@ -14,354 +14,71 @@ package helpers import ( - "fmt" "strings" - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/cast" + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/hugolib/paths" ) // PathSpec holds methods that decides how paths in URLs and files in Hugo should look like. type PathSpec struct { - BaseURL - - // If the baseURL contains a base path, e.g. https://example.com/docs, then "/docs" will be the BasePath. - // This will not be set if canonifyURLs is enabled. - BasePath string - - disablePathToLower bool - removePathAccents bool - uglyURLs bool - canonifyURLs bool - - Language *Language - Languages Languages - - // pagination path handling - paginatePath string - - theme string - - // Directories - contentDir string - themesDir string - layoutDir string - workingDir string - staticDirs []string - absContentDirs []types.KeyValueStr - - PublishDir string - - // The PathSpec looks up its config settings in both the current language - // and then in the global Viper config. - // Some settings, the settings listed below, does not make sense to be set - // on per-language-basis. We have no good way of protecting against this - // other than a "white-list". See language.go. - defaultContentLanguageInSubdir bool - defaultContentLanguage string - multilingual bool + *paths.Paths + *filesystems.BaseFs ProcessingStats *ProcessingStats // The file systems to use Fs *hugofs.Fs - // The fine grained filesystems in play (resources, content etc.). - BaseFs *hugofs.BaseFs - // The config provider to use Cfg config.Provider } -func (p PathSpec) String() string { - return fmt.Sprintf("PathSpec, language %q, prefix %q, multilingual: %T", p.Language.Lang, p.getLanguagePrefix(), p.multilingual) -} - -// NewPathSpec creats a new PathSpec from the given filesystems and Language. +// NewPathSpec creats a new PathSpec from the given filesystems and language. func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) { + return NewPathSpecWithBaseBaseFsProvided(fs, cfg, nil) +} - baseURLstr := cfg.GetString("baseURL") - baseURL, err := newBaseURLFromString(baseURLstr) - - if err != nil { - return nil, fmt.Errorf("Failed to create baseURL from %q: %s", baseURLstr, err) - } - - var staticDirs []string - - for i := -1; i <= 10; i++ { - staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...) - } - - var ( - lang string - language *Language - languages Languages - ) - - if l, ok := cfg.(*Language); ok { - language = l - lang = l.Lang - - } - - if l, ok := cfg.Get("languagesSorted").(Languages); ok { - languages = l - } - - defaultContentLanguage := cfg.GetString("defaultContentLanguage") - - // We will eventually pull out this badly placed path logic. - contentDir := cfg.GetString("contentDir") - workingDir := cfg.GetString("workingDir") - resourceDir := cfg.GetString("resourceDir") - publishDir := cfg.GetString("publishDir") - - if len(languages) == 0 { - // We have some old tests that does not test the entire chain, hence - // they have no languages. So create one so we get the proper filesystem. - languages = Languages{&Language{Lang: "en", ContentDir: contentDir}} - } +// NewPathSpecWithBaseBaseFsProvided creats a new PathSpec from the given filesystems and language. +// If an existing BaseFs is provided, parts of that is reused. +func NewPathSpecWithBaseBaseFsProvided(fs *hugofs.Fs, cfg config.Provider, baseBaseFs *filesystems.BaseFs) (*PathSpec, error) { - absPuslishDir := AbsPathify(workingDir, publishDir) - if !strings.HasSuffix(absPuslishDir, FilePathSeparator) { - absPuslishDir += FilePathSeparator - } - // If root, remove the second '/' - if absPuslishDir == "//" { - absPuslishDir = FilePathSeparator - } - absResourcesDir := AbsPathify(workingDir, resourceDir) - if !strings.HasSuffix(absResourcesDir, FilePathSeparator) { - absResourcesDir += FilePathSeparator - } - if absResourcesDir == "//" { - absResourcesDir = FilePathSeparator - } - - contentFs, absContentDirs, err := createContentFs(fs.Source, workingDir, defaultContentLanguage, languages) + p, err := paths.New(fs, cfg) if err != nil { return nil, err } - // Make sure we don't have any overlapping content dirs. That will never work. - for i, d1 := range absContentDirs { - for j, d2 := range absContentDirs { - if i == j { - continue - } - if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) { - return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2) - } + var options []func(*filesystems.BaseFs) error + if baseBaseFs != nil { + options = []func(*filesystems.BaseFs) error{ + filesystems.WithBaseFs(baseBaseFs), } } - - resourcesFs := afero.NewBasePathFs(fs.Source, absResourcesDir) - publishFs := afero.NewBasePathFs(fs.Destination, absPuslishDir) - - baseFs := &hugofs.BaseFs{ - ContentFs: contentFs, - ResourcesFs: resourcesFs, - PublishFs: publishFs, + bfs, err := filesystems.NewBase(p, options...) + if err != nil { + return nil, err } ps := &PathSpec{ - Fs: fs, - BaseFs: baseFs, - Cfg: cfg, - disablePathToLower: cfg.GetBool("disablePathToLower"), - removePathAccents: cfg.GetBool("removePathAccents"), - uglyURLs: cfg.GetBool("uglyURLs"), - canonifyURLs: cfg.GetBool("canonifyURLs"), - multilingual: cfg.GetBool("multilingual"), - Language: language, - Languages: languages, - defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"), - defaultContentLanguage: defaultContentLanguage, - paginatePath: cfg.GetString("paginatePath"), - BaseURL: baseURL, - contentDir: contentDir, - themesDir: cfg.GetString("themesDir"), - layoutDir: cfg.GetString("layoutDir"), - workingDir: workingDir, - staticDirs: staticDirs, - absContentDirs: absContentDirs, - theme: cfg.GetString("theme"), - ProcessingStats: NewProcessingStats(lang), + Paths: p, + BaseFs: bfs, + Fs: fs, + Cfg: cfg, + ProcessingStats: NewProcessingStats(p.Lang()), } - if !ps.canonifyURLs { - basePath := ps.BaseURL.url.Path + if !ps.CanonifyURLs { + basePath := ps.BaseURL.Path() if basePath != "" && basePath != "/" { ps.BasePath = basePath } } - // TODO(bep) remove this, eventually - ps.PublishDir = absPuslishDir - return ps, nil } -func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { - - if id >= 0 { - key = fmt.Sprintf("%s%d", key, id) - } - - var out []string - - sd := cfg.Get(key) - - if sds, ok := sd.(string); ok { - out = []string{sds} - } else if sd != nil { - out = cast.ToStringSlice(sd) - } - - return out -} - -func createContentFs(fs afero.Fs, - workingDir, - defaultContentLanguage string, - languages Languages) (afero.Fs, []types.KeyValueStr, error) { - - var contentLanguages Languages - var contentDirSeen = make(map[string]bool) - languageSet := make(map[string]bool) - - // The default content language needs to be first. - for _, language := range languages { - if language.Lang == defaultContentLanguage { - contentLanguages = append(contentLanguages, language) - contentDirSeen[language.ContentDir] = true - } - languageSet[language.Lang] = true - } - - for _, language := range languages { - if contentDirSeen[language.ContentDir] { - continue - } - if language.ContentDir == "" { - language.ContentDir = defaultContentLanguage - } - contentDirSeen[language.ContentDir] = true - contentLanguages = append(contentLanguages, language) - - } - - var absContentDirs []types.KeyValueStr - - fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs) - return fs, absContentDirs, err - -} - -func createContentOverlayFs(source afero.Fs, - workingDir string, - languages Languages, - languageSet map[string]bool, - absContentDirs *[]types.KeyValueStr) (afero.Fs, error) { - if len(languages) == 0 { - return source, nil - } - - language := languages[0] - - contentDir := language.ContentDir - if contentDir == "" { - panic("missing contentDir") - } - - absContentDir := AbsPathify(workingDir, language.ContentDir) - if !strings.HasSuffix(absContentDir, FilePathSeparator) { - absContentDir += FilePathSeparator - } - - // If root, remove the second '/' - if absContentDir == "//" { - absContentDir = FilePathSeparator - } - - if len(absContentDir) < 6 { - return nil, fmt.Errorf("invalid content dir %q: %s", absContentDir, ErrPathTooShort) - } - - *absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir}) - - overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir)) - if len(languages) == 1 { - return overlay, nil - } - - base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs) - if err != nil { - return nil, err - } - - return hugofs.NewLanguageCompositeFs(base, overlay), nil - -} - -// RelContentDir tries to create a path relative to the content root from -// the given filename. The return value is the path and language code. -func (p *PathSpec) RelContentDir(filename string) (string, string) { - for _, dir := range p.absContentDirs { - if strings.HasPrefix(filename, dir.Value) { - rel := strings.TrimPrefix(filename, dir.Value) - return strings.TrimPrefix(rel, FilePathSeparator), dir.Key - } - } - // Either not a content dir or already relative. - return filename, "" -} - -// ContentDirs returns all the content dirs (absolute paths). -func (p *PathSpec) ContentDirs() []types.KeyValueStr { - return p.absContentDirs -} - -// PaginatePath returns the configured root path used for paginator pages. -func (p *PathSpec) PaginatePath() string { - return p.paginatePath -} - -// ContentDir returns the configured workingDir. -func (p *PathSpec) ContentDir() string { - return p.contentDir -} - -// WorkingDir returns the configured workingDir. -func (p *PathSpec) WorkingDir() string { - return p.workingDir -} - -// StaticDirs returns the relative static dirs for the current configuration. -func (p *PathSpec) StaticDirs() []string { - return p.staticDirs -} - -// LayoutDir returns the relative layout dir in the current configuration. -func (p *PathSpec) LayoutDir() string { - return p.layoutDir -} - -// Theme returns the theme name if set. -func (p *PathSpec) Theme() string { - return p.theme -} - -// Theme returns the theme relative theme dir. -func (p *PathSpec) ThemesDir() string { - return p.themesDir -} - // PermalinkForBaseURL creates a permalink from the given link and baseURL. func (p *PathSpec) PermalinkForBaseURL(link, baseURL string) string { link = strings.TrimPrefix(link, "/") diff --git a/helpers/pathspec_test.go b/helpers/pathspec_test.go index dc2079e06..00dd9cd7b 100644 --- a/helpers/pathspec_test.go +++ b/helpers/pathspec_test.go @@ -1,4 +1,4 @@ -// Copyright 2016-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,20 +18,16 @@ import ( "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" + "github.com/gohugoio/hugo/langs" "github.com/stretchr/testify/require" ) func TestNewPathSpecFromConfig(t *testing.T) { - v := viper.New() - v.Set("contentDir", "content") - l := NewLanguage("no", v) + v := newTestCfg() + l := langs.NewLanguage("no", v) v.Set("disablePathToLower", true) v.Set("removePathAccents", true) v.Set("uglyURLs", true) - v.Set("multilingual", true) - v.Set("defaultContentLanguageInSubdir", true) - v.Set("defaultContentLanguage", "no") v.Set("canonifyURLs", true) v.Set("paginatePath", "side") v.Set("baseURL", "http://base.com") @@ -44,19 +40,15 @@ func TestNewPathSpecFromConfig(t *testing.T) { p, err := NewPathSpec(hugofs.NewMem(v), l) require.NoError(t, err) - require.True(t, p.canonifyURLs) - require.True(t, p.defaultContentLanguageInSubdir) - require.True(t, p.disablePathToLower) - require.True(t, p.multilingual) - require.True(t, p.removePathAccents) - require.True(t, p.uglyURLs) - require.Equal(t, "no", p.defaultContentLanguage) + require.True(t, p.CanonifyURLs) + require.True(t, p.DisablePathToLower) + require.True(t, p.RemovePathAccents) + require.True(t, p.UglyURLs) require.Equal(t, "no", p.Language.Lang) - require.Equal(t, "side", p.paginatePath) + require.Equal(t, "side", p.PaginatePath) require.Equal(t, "http://base.com", p.BaseURL.String()) - require.Equal(t, "thethemes", p.themesDir) - require.Equal(t, "thelayouts", p.layoutDir) - require.Equal(t, "thework", p.workingDir) - require.Equal(t, "thetheme", p.theme) + require.Equal(t, "thethemes", p.ThemesDir) + require.Equal(t, "thework", p.WorkingDir) + require.Equal(t, []string{"thetheme"}, p.Themes()) } diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index 215ae9188..fda1c9ea2 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -4,10 +4,11 @@ import ( "github.com/spf13/viper" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/langs" ) func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *PathSpec { - l := NewDefaultLanguage(v) + l := langs.NewDefaultLanguage(v) ps, _ := NewPathSpec(fs, l) return ps } @@ -15,7 +16,7 @@ func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *PathSpec { func newTestDefaultPathSpec(configKeyValues ...interface{}) *PathSpec { v := viper.New() fs := hugofs.NewMem(v) - cfg := newTestCfg(fs) + cfg := newTestCfgFor(fs) for i := 0; i < len(configKeyValues); i += 2 { cfg.Set(configKeyValues[i].(string), configKeyValues[i+1]) @@ -23,16 +24,24 @@ func newTestDefaultPathSpec(configKeyValues ...interface{}) *PathSpec { return newTestPathSpec(fs, cfg) } -func newTestCfg(fs *hugofs.Fs) *viper.Viper { - v := viper.New() - v.Set("contentDir", "content") - +func newTestCfgFor(fs *hugofs.Fs) *viper.Viper { + v := newTestCfg() v.SetFs(fs.Source) return v } +func newTestCfg() *viper.Viper { + v := viper.New() + v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") + return v +} + func newTestContentSpec() *ContentSpec { v := viper.New() spec, err := NewContentSpec(v) diff --git a/helpers/url.go b/helpers/url.go index ef08a7530..f167fd3d2 100644 --- a/helpers/url.go +++ b/helpers/url.go @@ -177,7 +177,7 @@ func (p *PathSpec) AbsURL(in string, addLanguage bool) string { } if addLanguage { - prefix := p.getLanguagePrefix() + prefix := p.GetLanguagePrefix() if prefix != "" { hasPrefix := false // avoid adding language prefix if already present @@ -200,38 +200,6 @@ func (p *PathSpec) AbsURL(in string, addLanguage bool) string { return MakePermalink(baseURL, in).String() } -func (p *PathSpec) getLanguagePrefix() string { - if !p.multilingual { - return "" - } - - defaultLang := p.defaultContentLanguage - defaultInSubDir := p.defaultContentLanguageInSubdir - - currentLang := p.Language.Lang - if currentLang == "" || (currentLang == defaultLang && !defaultInSubDir) { - return "" - } - return currentLang -} - -// GetLangSubDir returns the given language's subdir if needed. -func (p *PathSpec) GetLangSubDir(lang string) string { - if !p.multilingual { - return "" - } - - if p.Languages.IsMultihost() { - return "" - } - - if lang == "" || (lang == p.defaultContentLanguage && !p.defaultContentLanguageInSubdir) { - return "" - } - - return lang -} - // IsAbsURL determines whether the given path points to an absolute URL. func IsAbsURL(path string) bool { url, err := url.Parse(path) @@ -246,7 +214,7 @@ func IsAbsURL(path string) bool { // Note: The result URL will not include the context root if canonifyURLs is enabled. func (p *PathSpec) RelURL(in string, addLanguage bool) string { baseURL := p.BaseURL.String() - canonifyURLs := p.canonifyURLs + canonifyURLs := p.CanonifyURLs if (!strings.HasPrefix(in, baseURL) && strings.HasPrefix(in, "http")) || strings.HasPrefix(in, "//") { return in } @@ -258,7 +226,7 @@ func (p *PathSpec) RelURL(in string, addLanguage bool) string { } if addLanguage { - prefix := p.getLanguagePrefix() + prefix := p.GetLanguagePrefix() if prefix != "" { hasPrefix := false // avoid adding language prefix if already present @@ -339,7 +307,7 @@ func (p *PathSpec) URLizeAndPrep(in string) string { // URLPrep applies misc sanitation to the given URL. func (p *PathSpec) URLPrep(in string) string { - if p.uglyURLs { + if p.UglyURLs { return Uglify(SanitizeURL(in)) } pretty := PrettifyURL(SanitizeURL(in)) diff --git a/helpers/url_test.go b/helpers/url_test.go index 0ca3c8df2..a2c945dfe 100644 --- a/helpers/url_test.go +++ b/helpers/url_test.go @@ -19,16 +19,15 @@ import ( "testing" "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" + "github.com/gohugoio/hugo/langs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestURLize(t *testing.T) { - v := viper.New() - v.Set("contentDir", "content") - l := NewDefaultLanguage(v) + v := newTestCfg() + l := langs.NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) tests := []struct { @@ -64,7 +63,7 @@ func TestAbsURL(t *testing.T) { } func doTestAbsURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, lang string) { - v := viper.New() + v := newTestCfg() v.Set("multilingual", multilingual) v.Set("defaultContentLanguage", "en") v.Set("defaultContentLanguageInSubdir", defaultInSubDir) @@ -90,7 +89,7 @@ func doTestAbsURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, for _, test := range tests { v.Set("baseURL", test.baseURL) v.Set("contentDir", "content") - l := NewLanguage(lang, v) + l := langs.NewLanguage(lang, v) p, _ := NewPathSpec(hugofs.NewMem(v), l) output := p.AbsURL(test.input, addLanguage) @@ -140,7 +139,7 @@ func TestRelURL(t *testing.T) { } func doTestRelURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, lang string) { - v := viper.New() + v := newTestCfg() v.Set("multilingual", multilingual) v.Set("defaultContentLanguage", "en") v.Set("defaultContentLanguageInSubdir", defaultInSubDir) @@ -168,8 +167,7 @@ func doTestRelURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, for i, test := range tests { v.Set("baseURL", test.baseURL) v.Set("canonifyURLs", test.canonify) - v.Set("contentDir", "content") - l := NewLanguage(lang, v) + l := langs.NewLanguage(lang, v) p, _ := NewPathSpec(hugofs.NewMem(v), l) output := p.RelURL(test.input, addLanguage) @@ -255,10 +253,9 @@ func TestURLPrep(t *testing.T) { } for i, d := range data { - v := viper.New() + v := newTestCfg() v.Set("uglyURLs", d.ugly) - v.Set("contentDir", "content") - l := NewDefaultLanguage(v) + l := langs.NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) output := p.URLPrep(d.input) diff --git a/hugofs/base_fs.go b/hugofs/base_fs.go deleted file mode 100644 index 77af66dfe..000000000 --- a/hugofs/base_fs.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugofs - -import ( - "github.com/spf13/afero" -) - -// BaseFs contains the core base filesystems used by Hugo. The name "base" is used -// to underline that even if they can be composites, they all have a base path set to a specific -// resource folder, e.g "/my-project/content". So, no absolute filenames needed. -type BaseFs struct { - // The filesystem used to capture content. This can be a composite and - // language aware file system. - ContentFs afero.Fs - - // The filesystem used to store resources (processed images etc.). - // This usually maps to /my-project/resources. - ResourcesFs afero.Fs - - // The filesystem used to publish the rendered site. - // This usually maps to /my-project/public. - PublishFs afero.Fs -} diff --git a/hugofs/noop_fs.go b/hugofs/noop_fs.go new file mode 100644 index 000000000..2d06622e4 --- /dev/null +++ b/hugofs/noop_fs.go @@ -0,0 +1,79 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "errors" + "os" + "time" + + "github.com/spf13/afero" +) + +var ( + noOpErr = errors.New("this is a filesystem that does nothing and this operation is not supported") + _ afero.Fs = (*noOpFs)(nil) + NoOpFs = &noOpFs{} +) + +type noOpFs struct { +} + +func (fs noOpFs) Create(name string) (afero.File, error) { + return nil, noOpErr +} + +func (fs noOpFs) Mkdir(name string, perm os.FileMode) error { + return noOpErr +} + +func (fs noOpFs) MkdirAll(path string, perm os.FileMode) error { + return noOpErr +} + +func (fs noOpFs) Open(name string) (afero.File, error) { + return nil, os.ErrNotExist +} + +func (fs noOpFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + return nil, os.ErrNotExist +} + +func (fs noOpFs) Remove(name string) error { + return noOpErr +} + +func (fs noOpFs) RemoveAll(path string) error { + return noOpErr +} + +func (fs noOpFs) Rename(oldname string, newname string) error { + return noOpErr +} + +func (fs noOpFs) Stat(name string) (os.FileInfo, error) { + return nil, os.ErrNotExist +} + +func (fs noOpFs) Name() string { + return "noOpFs" +} + +func (fs noOpFs) Chmod(name string, mode os.FileMode) error { + return noOpErr +} + +func (fs noOpFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + return noOpErr +} diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go new file mode 100644 index 000000000..59f49f3a9 --- /dev/null +++ b/hugofs/rootmapping_fs.go @@ -0,0 +1,180 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "os" + "path/filepath" + "strings" + "time" + + radix "github.com/hashicorp/go-immutable-radix" + "github.com/spf13/afero" +) + +var filepathSeparator = string(filepath.Separator) + +// A RootMappingFs maps several roots into one. Note that the root of this filesystem +// is directories only, and they will be returned in Readdir and Readdirnames +// in the order given. +type RootMappingFs struct { + afero.Fs + rootMapToReal *radix.Node + virtualRoots []string +} + +type rootMappingFile struct { + afero.File + fs *RootMappingFs + name string +} + +type rootMappingFileInfo struct { + name string +} + +func (fi *rootMappingFileInfo) Name() string { + return fi.name +} + +func (fi *rootMappingFileInfo) Size() int64 { + panic("not implemented") +} + +func (fi *rootMappingFileInfo) Mode() os.FileMode { + return os.ModeDir +} + +func (fi *rootMappingFileInfo) ModTime() time.Time { + panic("not implemented") +} + +func (fi *rootMappingFileInfo) IsDir() bool { + return true +} + +func (fi *rootMappingFileInfo) Sys() interface{} { + return nil +} + +func newRootMappingDirFileInfo(name string) *rootMappingFileInfo { + return &rootMappingFileInfo{name: name} +} + +// NewRootMappingFs creates a new RootMappingFs on top of the provided with +// a list of from, to string pairs of root mappings. +// Note that 'from' represents a virtual root that maps to the actual filename in 'to'. +func NewRootMappingFs(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) { + rootMapToReal := radix.New().Txn() + var virtualRoots []string + + for i := 0; i < len(fromTo); i += 2 { + vr := filepath.Clean(fromTo[i]) + rr := filepath.Clean(fromTo[i+1]) + + // We need to preserve the original order for Readdir + virtualRoots = append(virtualRoots, vr) + + rootMapToReal.Insert([]byte(vr), rr) + } + + return &RootMappingFs{Fs: fs, + virtualRoots: virtualRoots, + rootMapToReal: rootMapToReal.Commit().Root()}, nil +} + +func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) { + if fs.isRoot(name) { + return newRootMappingDirFileInfo(name), nil + } + realName := fs.realName(name) + return fs.Fs.Stat(realName) +} + +func (fs *RootMappingFs) isRoot(name string) bool { + return name == "" || name == filepathSeparator + +} + +func (fs *RootMappingFs) Open(name string) (afero.File, error) { + if fs.isRoot(name) { + return &rootMappingFile{name: name, fs: fs}, nil + } + realName := fs.realName(name) + f, err := fs.Fs.Open(realName) + if err != nil { + return nil, err + } + return &rootMappingFile{File: f, name: name, fs: fs}, nil +} + +func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + if fs.isRoot(name) { + return newRootMappingDirFileInfo(name), false, nil + } + name = fs.realName(name) + if ls, ok := fs.Fs.(afero.Lstater); ok { + return ls.LstatIfPossible(name) + } + fi, err := fs.Stat(name) + return fi, false, err +} + +func (fs *RootMappingFs) realName(name string) string { + key, val, found := fs.rootMapToReal.LongestPrefix([]byte(filepath.Clean(name))) + if !found { + return name + } + keystr := string(key) + + return filepath.Join(val.(string), strings.TrimPrefix(name, keystr)) +} + +func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { + if f.File == nil { + dirsn := make([]os.FileInfo, 0) + for i := 0; i < len(f.fs.virtualRoots); i++ { + if count != -1 && i >= count { + break + } + dirsn = append(dirsn, newRootMappingDirFileInfo(f.fs.virtualRoots[i])) + } + return dirsn, nil + } + return f.File.Readdir(count) + +} + +func (f *rootMappingFile) Readdirnames(count int) ([]string, error) { + dirs, err := f.Readdir(count) + if err != nil { + return nil, err + } + dirss := make([]string, len(dirs)) + for i, d := range dirs { + dirss[i] = d.Name() + } + return dirss, nil +} + +func (f *rootMappingFile) Name() string { + return f.name +} + +func (f *rootMappingFile) Close() error { + if f.File == nil { + return nil + } + return f.File.Close() +} diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go new file mode 100644 index 000000000..a84f41151 --- /dev/null +++ b/hugofs/rootmapping_fs_test.go @@ -0,0 +1,93 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestRootMappingFsRealName(t *testing.T) { + assert := require.New(t) + fs := afero.NewMemMapFs() + + rfs, err := NewRootMappingFs(fs, "f1", "f1t", "f2", "f2t") + assert.NoError(err) + + assert.Equal(filepath.FromSlash("f1t/foo/file.txt"), rfs.realName(filepath.Join("f1", "foo", "file.txt"))) + +} + +func TestRootMappingFsDirnames(t *testing.T) { + assert := require.New(t) + fs := afero.NewMemMapFs() + + testfile := "myfile.txt" + assert.NoError(fs.Mkdir("f1t", 0755)) + assert.NoError(fs.Mkdir("f2t", 0755)) + assert.NoError(fs.Mkdir("f3t", 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("f2t", testfile), []byte("some content"), 0755)) + + rfs, err := NewRootMappingFs(fs, "bf1", "f1t", "cf2", "f2t", "af3", "f3t") + assert.NoError(err) + + fif, err := rfs.Stat(filepath.Join("cf2", testfile)) + assert.NoError(err) + assert.Equal("myfile.txt", fif.Name()) + + root, err := rfs.Open(filepathSeparator) + assert.NoError(err) + + dirnames, err := root.Readdirnames(-1) + assert.NoError(err) + assert.Equal([]string{"bf1", "cf2", "af3"}, dirnames) + +} + +func TestRootMappingFsOs(t *testing.T) { + assert := require.New(t) + fs := afero.NewOsFs() + + d, err := ioutil.TempDir("", "hugo-root-mapping") + assert.NoError(err) + defer func() { + os.RemoveAll(d) + }() + + testfile := "myfile.txt" + assert.NoError(fs.Mkdir(filepath.Join(d, "f1t"), 0755)) + assert.NoError(fs.Mkdir(filepath.Join(d, "f2t"), 0755)) + assert.NoError(fs.Mkdir(filepath.Join(d, "f3t"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755)) + + rfs, err := NewRootMappingFs(fs, "bf1", filepath.Join(d, "f1t"), "cf2", filepath.Join(d, "f2t"), "af3", filepath.Join(d, "f3t")) + assert.NoError(err) + + fif, err := rfs.Stat(filepath.Join("cf2", testfile)) + assert.NoError(err) + assert.Equal("myfile.txt", fif.Name()) + + root, err := rfs.Open(filepathSeparator) + assert.NoError(err) + + dirnames, err := root.Readdirnames(-1) + assert.NoError(err) + assert.Equal([]string{"bf1", "cf2", "af3"}, dirnames) + +} diff --git a/hugolib/alias_test.go b/hugolib/alias_test.go index d20409512..04c5b4358 100644 --- a/hugolib/alias_test.go +++ b/hugolib/alias_test.go @@ -18,6 +18,8 @@ import ( "runtime" "testing" + "github.com/gohugoio/hugo/common/loggers" + "github.com/stretchr/testify/require" ) @@ -97,7 +99,7 @@ func TestAliasTemplate(t *testing.T) { } func TestTargetPathHTMLRedirectAlias(t *testing.T) { - h := newAliasHandler(nil, newErrorLogger(), false) + h := newAliasHandler(nil, loggers.NewErrorLogger(), false) errIsNilForThisOS := runtime.GOOS != "windows" diff --git a/hugolib/case_insensitive_test.go b/hugolib/case_insensitive_test.go index 52ef198a5..f3ba5f933 100644 --- a/hugolib/case_insensitive_test.go +++ b/hugolib/case_insensitive_test.go @@ -19,8 +19,9 @@ import ( "strings" "testing" - "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" + + "github.com/gohugoio/hugo/deps" "github.com/spf13/afero" "github.com/stretchr/testify/require" ) diff --git a/hugolib/config.go b/hugolib/config.go index 73ba84686..dec5b870d 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -16,11 +16,14 @@ package hugolib import ( "errors" "fmt" - "path/filepath" + + "github.com/gohugoio/hugo/hugolib/paths" "io" "strings" + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config/privacy" "github.com/gohugoio/hugo/config/services" @@ -81,6 +84,8 @@ func LoadConfigDefault(fs afero.Fs) (*viper.Viper, error) { return v, err } +var ErrNoConfigFile = errors.New("Unable to locate Config file. Perhaps you need to create a new site.\n Run `hugo help new` for details.\n") + // LoadConfig loads Hugo configuration into a new Viper and then adds // a set of defaults. func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provider) error) (*viper.Viper, []string, error) { @@ -100,41 +105,50 @@ func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provid v.SetConfigFile(configFilenames[0]) v.AddConfigPath(d.Path) + var configFileErr error + err := v.ReadInConfig() if err != nil { if _, ok := err.(viper.ConfigParseError); ok { return nil, configFiles, err } - return nil, configFiles, fmt.Errorf("Unable to locate Config file. Perhaps you need to create a new site.\n Run `hugo help new` for details. (%s)\n", err) + configFileErr = ErrNoConfigFile } - if cf := v.ConfigFileUsed(); cf != "" { - configFiles = append(configFiles, cf) - } + if configFileErr == nil { - for _, configFile := range configFilenames[1:] { - var r io.Reader - var err error - if r, err = fs.Open(configFile); err != nil { - return nil, configFiles, fmt.Errorf("Unable to open Config file.\n (%s)\n", err) + if cf := v.ConfigFileUsed(); cf != "" { + configFiles = append(configFiles, cf) } - if err = v.MergeConfig(r); err != nil { - return nil, configFiles, fmt.Errorf("Unable to parse/merge Config file (%s).\n (%s)\n", configFile, err) + + for _, configFile := range configFilenames[1:] { + var r io.Reader + var err error + if r, err = fs.Open(configFile); err != nil { + return nil, configFiles, fmt.Errorf("Unable to open Config file.\n (%s)\n", err) + } + if err = v.MergeConfig(r); err != nil { + return nil, configFiles, fmt.Errorf("Unable to parse/merge Config file (%s).\n (%s)\n", configFile, err) + } + configFiles = append(configFiles, configFile) } - configFiles = append(configFiles, configFile) + } if err := loadDefaultSettingsFor(v); err != nil { return v, configFiles, err } - themeConfigFile, err := loadThemeConfig(d, v) - if err != nil { - return v, configFiles, err - } + if configFileErr == nil { - if themeConfigFile != "" { - configFiles = append(configFiles, themeConfigFile) + themeConfigFiles, err := loadThemeConfig(d, v) + if err != nil { + return v, configFiles, err + } + + if len(themeConfigFiles) > 0 { + configFiles = append(configFiles, themeConfigFiles...) + } } // We create languages based on the settings, so we need to make sure that @@ -149,11 +163,11 @@ func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provid return v, configFiles, err } - return v, configFiles, nil + return v, configFiles, configFileErr } -func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error { +func loadLanguageSettings(cfg config.Provider, oldLangs langs.Languages) error { defaultLang := cfg.GetString("defaultContentLanguage") @@ -182,14 +196,14 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error } var ( - langs helpers.Languages - err error + languages2 langs.Languages + err error ) if len(languages) == 0 { - langs = append(langs, helpers.NewDefaultLanguage(cfg)) + languages2 = append(languages2, langs.NewDefaultLanguage(cfg)) } else { - langs, err = toSortedLanguages(cfg, languages) + languages2, err = toSortedLanguages(cfg, languages) if err != nil { return fmt.Errorf("Failed to parse multilingual config: %s", err) } @@ -201,10 +215,10 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error // The validation below isn't complete, but should cover the most // important cases. var invalid bool - if langs.IsMultihost() != oldLangs.IsMultihost() { + if languages2.IsMultihost() != oldLangs.IsMultihost() { invalid = true } else { - if langs.IsMultihost() && len(langs) != len(oldLangs) { + if languages2.IsMultihost() && len(languages2) != len(oldLangs) { invalid = true } } @@ -213,10 +227,10 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error return errors.New("language change needing a server restart detected") } - if langs.IsMultihost() { + if languages2.IsMultihost() { // We need to transfer any server baseURL to the new language for i, ol := range oldLangs { - nl := langs[i] + nl := languages2[i] nl.Set("baseURL", ol.GetString("baseURL")) } } @@ -225,7 +239,7 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error // The defaultContentLanguage is something the user has to decide, but it needs // to match a language in the language definition list. langExists := false - for _, lang := range langs { + for _, lang := range languages2 { if lang.Lang == defaultLang { langExists = true break @@ -236,10 +250,10 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error return fmt.Errorf("site config value %q for defaultContentLanguage does not match any language definition", defaultLang) } - cfg.Set("languagesSorted", langs) - cfg.Set("multilingual", len(langs) > 1) + cfg.Set("languagesSorted", languages2) + cfg.Set("multilingual", len(languages2) > 1) - multihost := langs.IsMultihost() + multihost := languages2.IsMultihost() if multihost { cfg.Set("defaultContentLanguageInSubdir", true) @@ -250,7 +264,7 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error // The baseURL may be provided at the language level. If that is true, // then every language must have a baseURL. In this case we always render // to a language sub folder, which is then stripped from all the Permalink URLs etc. - for _, l := range langs { + for _, l := range languages2 { burl := l.GetLocal("baseURL") if burl == nil { return errors.New("baseURL must be set on all or none of the languages") @@ -262,49 +276,32 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error return nil } -func loadThemeConfig(d ConfigSourceDescriptor, v1 *viper.Viper) (string, error) { +func loadThemeConfig(d ConfigSourceDescriptor, v1 *viper.Viper) ([]string, error) { + themesDir := paths.AbsPathify(d.WorkingDir, v1.GetString("themesDir")) + themes := config.GetStringSlicePreserveString(v1, "theme") - theme := v1.GetString("theme") - if theme == "" { - return "", nil + // CollectThemes(fs afero.Fs, themesDir string, themes []strin + themeConfigs, err := paths.CollectThemes(d.Fs, themesDir, themes) + if err != nil { + return nil, err } - - themesDir := helpers.AbsPathify(d.WorkingDir, v1.GetString("themesDir")) - configDir := filepath.Join(themesDir, theme) - - var ( - configPath string - exists bool - err error - ) - - // Viper supports more, but this is the sub-set supported by Hugo. - for _, configFormats := range []string{"toml", "yaml", "yml", "json"} { - configPath = filepath.Join(configDir, "config."+configFormats) - exists, err = helpers.Exists(configPath, d.Fs) - if err != nil { - return "", err - } - if exists { - break + v1.Set("allThemes", themeConfigs) + + var configFilenames []string + for _, tc := range themeConfigs { + if tc.ConfigFilename != "" { + configFilenames = append(configFilenames, tc.ConfigFilename) + if err := applyThemeConfig(v1, tc); err != nil { + return nil, err + } } } - if !exists { - // No theme config set. - return "", nil - } + return configFilenames, nil - v2 := viper.New() - v2.SetFs(d.Fs) - v2.AutomaticEnv() - v2.SetEnvPrefix("hugo") - v2.SetConfigFile(configPath) +} - err = v2.ReadInConfig() - if err != nil { - return "", err - } +func applyThemeConfig(v1 *viper.Viper, theme paths.ThemeConfig) error { const ( paramsKey = "params" @@ -312,11 +309,13 @@ func loadThemeConfig(d ConfigSourceDescriptor, v1 *viper.Viper) (string, error) menuKey = "menu" ) + v2 := theme.Cfg + for _, key := range []string{paramsKey, "outputformats", "mediatypes"} { mergeStringMapKeepLeft("", key, v1, v2) } - themeLower := strings.ToLower(theme) + themeLower := strings.ToLower(theme.Name) themeParamsNamespace := paramsKey + "." + themeLower // Set namespaced params @@ -371,11 +370,11 @@ func loadThemeConfig(d ConfigSourceDescriptor, v1 *viper.Viper) (string, error) } } - return v2.ConfigFileUsed(), nil + return nil } -func mergeStringMapKeepLeft(rootKey, key string, v1, v2 *viper.Viper) { +func mergeStringMapKeepLeft(rootKey, key string, v1, v2 config.Provider) { if !v2.IsSet(key) { return } diff --git a/hugolib/datafiles_test.go b/hugolib/datafiles_test.go index cd1ad8411..8b2dc8c0f 100644 --- a/hugolib/datafiles_test.go +++ b/hugolib/datafiles_test.go @@ -19,6 +19,8 @@ import ( "strings" "testing" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/deps" "fmt" @@ -322,7 +324,7 @@ func doTestDataDirImpl(t *testing.T, dd dataDir, expected interface{}, configKey } var ( - logger = newErrorLogger() + logger = loggers.NewErrorLogger() depsCfg = deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: logger} ) diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go new file mode 100644 index 000000000..deecd69a5 --- /dev/null +++ b/hugolib/filesystems/basefs.go @@ -0,0 +1,644 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package filesystems provides the fine grained file systems used by Hugo. These +// are typically virtual filesystems that are composites of project and theme content. +package filesystems + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/config" + + "github.com/gohugoio/hugo/hugofs" + + "fmt" + + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/hugolib/paths" + "github.com/gohugoio/hugo/langs" + "github.com/spf13/afero" +) + +// When we create a virtual filesystem with data and i18n bundles for the project and the themes, +// this is the name of the project's virtual root. It got it's funky name to make sure +// (or very unlikely) that it collides with a theme name. +const projectVirtualFolder = "__h__project" + +var filePathSeparator = string(filepath.Separator) + +// BaseFs contains the core base filesystems used by Hugo. The name "base" is used +// to underline that even if they can be composites, they all have a base path set to a specific +// resource folder, e.g "/my-project/content". So, no absolute filenames needed. +type BaseFs struct { + // TODO(bep) make this go away + AbsContentDirs []types.KeyValueStr + + // The filesystem used to capture content. This can be a composite and + // language aware file system. + ContentFs afero.Fs + + // SourceFilesystems contains the different source file systems. + *SourceFilesystems + + // The filesystem used to store resources (processed images etc.). + // This usually maps to /my-project/resources. + ResourcesFs afero.Fs + + // The filesystem used to publish the rendered site. + // This usually maps to /my-project/public. + PublishFs afero.Fs + + themeFs afero.Fs + + // TODO(bep) improve the "theme interaction" + AbsThemeDirs []string +} + +// RelContentDir tries to create a path relative to the content root from +// the given filename. The return value is the path and language code. +func (b *BaseFs) RelContentDir(filename string) (string, string) { + for _, dir := range b.AbsContentDirs { + if strings.HasPrefix(filename, dir.Value) { + rel := strings.TrimPrefix(filename, dir.Value) + return strings.TrimPrefix(rel, filePathSeparator), dir.Key + } + } + // Either not a content dir or already relative. + return filename, "" +} + +// IsContent returns whether the given filename is in the content filesystem. +func (b *BaseFs) IsContent(filename string) bool { + for _, dir := range b.AbsContentDirs { + if strings.HasPrefix(filename, dir.Value) { + return true + } + } + return false +} + +// SourceFilesystems contains the different source file systems. These can be +// composite file systems (theme and project etc.), and they have all root +// set to the source type the provides: data, i18n, static, layouts. +type SourceFilesystems struct { + Data *SourceFilesystem + I18n *SourceFilesystem + Layouts *SourceFilesystem + Archetypes *SourceFilesystem + + // When in multihost we have one static filesystem per language. The sync + // static files is currently done outside of the Hugo build (where there is + // a concept of a site per language). + // When in non-multihost mode there will be one entry in this map with a blank key. + Static map[string]*SourceFilesystem +} + +// A SourceFilesystem holds the filesystem for a given source type in Hugo (data, +// i18n, layouts, static) and additional metadata to be able to use that filesystem +// in server mode. +type SourceFilesystem struct { + Fs afero.Fs + + Dirnames []string + + // When syncing a source folder to the target (e.g. /public), this may + // be set to publish into a subfolder. This is used for static syncing + // in multihost mode. + PublishFolder string +} + +// IsStatic returns true if the given filename is a member of one of the static +// filesystems. +func (s SourceFilesystems) IsStatic(filename string) bool { + for _, staticFs := range s.Static { + if staticFs.Contains(filename) { + return true + } + } + return false +} + +// IsLayout returns true if the given filename is a member of the layouts filesystem. +func (s SourceFilesystems) IsLayout(filename string) bool { + return s.Layouts.Contains(filename) +} + +// IsData returns true if the given filename is a member of the data filesystem. +func (s SourceFilesystems) IsData(filename string) bool { + return s.Data.Contains(filename) +} + +// IsI18n returns true if the given filename is a member of the i18n filesystem. +func (s SourceFilesystems) IsI18n(filename string) bool { + return s.I18n.Contains(filename) +} + +// MakeStaticPathRelative makes an absolute static filename into a relative one. +// It will return an empty string if the filename is not a member of a static filesystem. +func (s SourceFilesystems) MakeStaticPathRelative(filename string) string { + for _, staticFs := range s.Static { + rel := staticFs.MakePathRelative(filename) + if rel != "" { + return rel + } + } + return "" +} + +// MakePathRelative creates a relative path from the given filename. +// It will return an empty string if the filename is not a member of this filesystem. +func (d *SourceFilesystem) MakePathRelative(filename string) string { + for _, currentPath := range d.Dirnames { + if strings.HasPrefix(filename, currentPath) { + return strings.TrimPrefix(filename, currentPath) + } + } + return "" +} + +// Contains returns whether the given filename is a member of the current filesystem. +func (d *SourceFilesystem) Contains(filename string) bool { + for _, dir := range d.Dirnames { + if strings.HasPrefix(filename, dir) { + return true + } + } + return false +} + +// WithBaseFs allows reuse of some potentially expensive to create parts that remain +// the same across sites/languages. +func WithBaseFs(b *BaseFs) func(*BaseFs) error { + return func(bb *BaseFs) error { + bb.themeFs = b.themeFs + bb.AbsThemeDirs = b.AbsThemeDirs + return nil + } +} + +// NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase +func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { + fs := p.Fs + + resourcesFs := afero.NewBasePathFs(fs.Source, p.AbsResourcesDir) + publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir) + + contentFs, absContentDirs, err := createContentFs(fs.Source, p.WorkingDir, p.DefaultContentLanguage, p.Languages) + if err != nil { + return nil, err + } + + // Make sure we don't have any overlapping content dirs. That will never work. + for i, d1 := range absContentDirs { + for j, d2 := range absContentDirs { + if i == j { + continue + } + if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) { + return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2) + } + } + } + + b := &BaseFs{ + AbsContentDirs: absContentDirs, + ContentFs: contentFs, + ResourcesFs: resourcesFs, + PublishFs: publishFs, + } + + for _, opt := range options { + if err := opt(b); err != nil { + return nil, err + } + } + + builder := newSourceFilesystemsBuilder(p, b) + sourceFilesystems, err := builder.Build() + if err != nil { + return nil, err + } + + b.SourceFilesystems = sourceFilesystems + b.themeFs = builder.themeFs + b.AbsThemeDirs = builder.absThemeDirs + + return b, nil +} + +type sourceFilesystemsBuilder struct { + p *paths.Paths + result *SourceFilesystems + themeFs afero.Fs + hasTheme bool + absThemeDirs []string +} + +func newSourceFilesystemsBuilder(p *paths.Paths, b *BaseFs) *sourceFilesystemsBuilder { + return &sourceFilesystemsBuilder{p: p, themeFs: b.themeFs, absThemeDirs: b.AbsThemeDirs, result: &SourceFilesystems{}} +} + +func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { + if b.themeFs == nil && b.p.ThemeSet() { + themeFs, absThemeDirs, err := createThemesOverlayFs(b.p) + if err != nil { + return nil, err + } + if themeFs == nil { + panic("createThemesFs returned nil") + } + b.themeFs = themeFs + b.absThemeDirs = absThemeDirs + + } + + b.hasTheme = len(b.absThemeDirs) > 0 + + sfs, err := b.createRootMappingFs("dataDir", "data") + if err != nil { + return nil, err + } + b.result.Data = sfs + + sfs, err = b.createRootMappingFs("i18nDir", "i18n") + if err != nil { + return nil, err + } + b.result.I18n = sfs + + sfs, err = b.createFs("layoutDir", "layouts") + if err != nil { + return nil, err + } + b.result.Layouts = sfs + + sfs, err = b.createFs("archetypeDir", "archetypes") + if err != nil { + return nil, err + } + b.result.Archetypes = sfs + + err = b.createStaticFs() + if err != nil { + return nil, err + } + + return b.result, nil +} + +func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*SourceFilesystem, error) { + s := &SourceFilesystem{} + dir := b.p.Cfg.GetString(dirKey) + if dir == "" { + return s, fmt.Errorf("config %q not set", dirKey) + } + + var fs afero.Fs + + absDir := b.p.AbsPathify(dir) + if b.existsInSource(absDir) { + fs = afero.NewBasePathFs(b.p.Fs.Source, absDir) + s.Dirnames = []string{absDir} + } + + if b.hasTheme { + themeFolderFs := afero.NewBasePathFs(b.themeFs, themeFolder) + if fs == nil { + fs = themeFolderFs + } else { + fs = afero.NewCopyOnWriteFs(themeFolderFs, fs) + } + + for _, absThemeDir := range b.absThemeDirs { + absThemeFolderDir := filepath.Join(absThemeDir, themeFolder) + if b.existsInSource(absThemeFolderDir) { + s.Dirnames = append(s.Dirnames, absThemeFolderDir) + } + } + } + + if fs == nil { + s.Fs = hugofs.NoOpFs + } else { + s.Fs = afero.NewReadOnlyFs(fs) + } + + return s, nil +} + +// Used for data, i18n -- we cannot use overlay filsesystems for those, but we need +// to keep a strict order. +func (b *sourceFilesystemsBuilder) createRootMappingFs(dirKey, themeFolder string) (*SourceFilesystem, error) { + s := &SourceFilesystem{} + + projectDir := b.p.Cfg.GetString(dirKey) + if projectDir == "" { + return nil, fmt.Errorf("config %q not set", dirKey) + } + + var fromTo []string + to := b.p.AbsPathify(projectDir) + + if b.existsInSource(to) { + s.Dirnames = []string{to} + fromTo = []string{projectVirtualFolder, to} + } + + for _, theme := range b.p.AllThemes { + to := b.p.AbsPathify(filepath.Join(b.p.ThemesDir, theme.Name, themeFolder)) + if b.existsInSource(to) { + s.Dirnames = append(s.Dirnames, to) + from := theme + fromTo = append(fromTo, from.Name, to) + } + } + + if len(fromTo) == 0 { + s.Fs = hugofs.NoOpFs + return s, nil + } + + fs, err := hugofs.NewRootMappingFs(b.p.Fs.Source, fromTo...) + if err != nil { + return nil, err + } + + s.Fs = afero.NewReadOnlyFs(fs) + + return s, nil + +} + +func (b *sourceFilesystemsBuilder) existsInSource(abspath string) bool { + exists, _ := afero.Exists(b.p.Fs.Source, abspath) + return exists +} + +func (b *sourceFilesystemsBuilder) createStaticFs() error { + isMultihost := b.p.Cfg.GetBool("multihost") + ms := make(map[string]*SourceFilesystem) + b.result.Static = ms + + if isMultihost { + for _, l := range b.p.Languages { + s := &SourceFilesystem{PublishFolder: l.Lang} + staticDirs := removeDuplicatesKeepRight(getStaticDirs(l)) + if len(staticDirs) == 0 { + continue + } + + for _, dir := range staticDirs { + absDir := b.p.AbsPathify(dir) + if !b.existsInSource(absDir) { + continue + } + + s.Dirnames = append(s.Dirnames, absDir) + } + + fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames) + if err != nil { + return err + } + + s.Fs = fs + ms[l.Lang] = s + + } + + return nil + } + + s := &SourceFilesystem{} + var staticDirs []string + + for _, l := range b.p.Languages { + staticDirs = append(staticDirs, getStaticDirs(l)...) + } + + staticDirs = removeDuplicatesKeepRight(staticDirs) + if len(staticDirs) == 0 { + return nil + } + + for _, dir := range staticDirs { + absDir := b.p.AbsPathify(dir) + if !b.existsInSource(absDir) { + continue + } + s.Dirnames = append(s.Dirnames, absDir) + } + + fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames) + if err != nil { + return err + } + + if b.hasTheme { + themeFolder := "static" + fs = afero.NewCopyOnWriteFs(afero.NewBasePathFs(b.themeFs, themeFolder), fs) + for _, absThemeDir := range b.absThemeDirs { + s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder)) + } + } + + s.Fs = fs + ms[""] = s + + return nil +} + +func getStaticDirs(cfg config.Provider) []string { + var staticDirs []string + for i := -1; i <= 10; i++ { + staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...) + } + return staticDirs +} + +func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { + + if id >= 0 { + key = fmt.Sprintf("%s%d", key, id) + } + + return config.GetStringSlicePreserveString(cfg, key) + +} + +func createContentFs(fs afero.Fs, + workingDir, + defaultContentLanguage string, + languages langs.Languages) (afero.Fs, []types.KeyValueStr, error) { + + var contentLanguages langs.Languages + var contentDirSeen = make(map[string]bool) + languageSet := make(map[string]bool) + + // The default content language needs to be first. + for _, language := range languages { + if language.Lang == defaultContentLanguage { + contentLanguages = append(contentLanguages, language) + contentDirSeen[language.ContentDir] = true + } + languageSet[language.Lang] = true + } + + for _, language := range languages { + if contentDirSeen[language.ContentDir] { + continue + } + if language.ContentDir == "" { + language.ContentDir = defaultContentLanguage + } + contentDirSeen[language.ContentDir] = true + contentLanguages = append(contentLanguages, language) + + } + + var absContentDirs []types.KeyValueStr + + fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs) + return fs, absContentDirs, err + +} + +func createContentOverlayFs(source afero.Fs, + workingDir string, + languages langs.Languages, + languageSet map[string]bool, + absContentDirs *[]types.KeyValueStr) (afero.Fs, error) { + if len(languages) == 0 { + return source, nil + } + + language := languages[0] + + contentDir := language.ContentDir + if contentDir == "" { + panic("missing contentDir") + } + + absContentDir := paths.AbsPathify(workingDir, language.ContentDir) + if !strings.HasSuffix(absContentDir, paths.FilePathSeparator) { + absContentDir += paths.FilePathSeparator + } + + // If root, remove the second '/' + if absContentDir == "//" { + absContentDir = paths.FilePathSeparator + } + + if len(absContentDir) < 6 { + return nil, fmt.Errorf("invalid content dir %q: Path is too short", absContentDir) + } + + *absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir}) + + overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir)) + if len(languages) == 1 { + return overlay, nil + } + + base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs) + if err != nil { + return nil, err + } + + return hugofs.NewLanguageCompositeFs(base, overlay), nil + +} + +func createThemesOverlayFs(p *paths.Paths) (afero.Fs, []string, error) { + + themes := p.AllThemes + + if len(themes) == 0 { + panic("AllThemes not set") + } + + themesDir := p.AbsPathify(p.ThemesDir) + if themesDir == "" { + return nil, nil, errors.New("no themes dir set") + } + + absPaths := make([]string, len(themes)) + + // The themes are ordered from left to right. We need to revert it to get the + // overlay logic below working as expected. + for i := 0; i < len(themes); i++ { + absPaths[i] = filepath.Join(themesDir, themes[len(themes)-1-i].Name) + } + + fs, err := createOverlayFs(p.Fs.Source, absPaths) + + return fs, absPaths, err + +} + +func createOverlayFs(source afero.Fs, absPaths []string) (afero.Fs, error) { + if len(absPaths) == 0 { + return hugofs.NoOpFs, nil + } + + if len(absPaths) == 1 { + return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])), nil + } + + base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])) + overlay, err := createOverlayFs(source, absPaths[1:]) + if err != nil { + return nil, err + } + + return afero.NewCopyOnWriteFs(base, overlay), nil +} + +func removeDuplicatesKeepRight(in []string) []string { + seen := make(map[string]bool) + var out []string + for i := len(in) - 1; i >= 0; i-- { + v := in[i] + if seen[v] { + continue + } + out = append([]string{v}, out...) + seen[v] = true + } + + return out +} + +func printFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return + } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + if info != nil && !info.IsDir() { + s := path + if lang, ok := info.(hugofs.LanguageAnnouncer); ok { + s = s + "\tLANG: " + lang.Lang() + } + if fp, ok := info.(hugofs.FilePather); ok { + s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir() + } + fmt.Fprintln(w, " ", s) + } + return nil + }) +} diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go new file mode 100644 index 000000000..ea09cd8fd --- /dev/null +++ b/hugolib/filesystems/basefs_test.go @@ -0,0 +1,170 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filesystems + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugolib/paths" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestNewBaseFs(t *testing.T) { + assert := require.New(t) + v := viper.New() + + fs := hugofs.NewMem(v) + + themes := []string{"btheme", "atheme"} + + workingDir := filepath.FromSlash("/my/work") + v.Set("workingDir", workingDir) + v.Set("themesDir", "themes") + v.Set("theme", themes[:1]) + + // Write some data to the themes + for _, theme := range themes { + for _, dir := range []string{"i18n", "data"} { + base := filepath.Join(workingDir, "themes", theme, dir) + fs.Source.Mkdir(base, 0755) + afero.WriteFile(fs.Source, filepath.Join(base, fmt.Sprintf("theme-file-%s-%s.txt", theme, dir)), []byte(fmt.Sprintf("content:%s:%s", theme, dir)), 0755) + } + } + + afero.WriteFile(fs.Source, filepath.Join(workingDir, "themes", "btheme", "config.toml"), []byte(` +theme = ["atheme"] +`), 0755) + + setConfigAndWriteSomeFilesTo(fs.Source, v, "contentDir", "mycontent", 3) + setConfigAndWriteSomeFilesTo(fs.Source, v, "i18nDir", "myi18n", 4) + setConfigAndWriteSomeFilesTo(fs.Source, v, "layoutDir", "mylayouts", 5) + setConfigAndWriteSomeFilesTo(fs.Source, v, "staticDir", "mystatic", 6) + setConfigAndWriteSomeFilesTo(fs.Source, v, "dataDir", "mydata", 7) + setConfigAndWriteSomeFilesTo(fs.Source, v, "archetypeDir", "myarchetypes", 8) + + p, err := paths.New(fs, v) + assert.NoError(err) + + bfs, err := NewBase(p) + assert.NoError(err) + assert.NotNil(bfs) + + root, err := bfs.I18n.Fs.Open("") + assert.NoError(err) + dirnames, err := root.Readdirnames(-1) + assert.NoError(err) + assert.Equal([]string{projectVirtualFolder, "btheme", "atheme"}, dirnames) + ff, err := bfs.I18n.Fs.Open("myi18n") + assert.NoError(err) + _, err = ff.Readdirnames(-1) + assert.NoError(err) + + root, err = bfs.Data.Fs.Open("") + assert.NoError(err) + dirnames, err = root.Readdirnames(-1) + assert.NoError(err) + assert.Equal([]string{projectVirtualFolder, "btheme", "atheme"}, dirnames) + ff, err = bfs.I18n.Fs.Open("mydata") + assert.NoError(err) + _, err = ff.Readdirnames(-1) + assert.NoError(err) + + checkFileCount(bfs.ContentFs, "", assert, 3) + checkFileCount(bfs.I18n.Fs, "", assert, 6) // 4 + 2 themes + checkFileCount(bfs.Layouts.Fs, "", assert, 5) + checkFileCount(bfs.Static[""].Fs, "", assert, 6) + checkFileCount(bfs.Data.Fs, "", assert, 9) // 7 + 2 themes + checkFileCount(bfs.Archetypes.Fs, "", assert, 8) + + assert.Equal([]string{filepath.FromSlash("/my/work/mydata"), filepath.FromSlash("/my/work/themes/btheme/data"), filepath.FromSlash("/my/work/themes/atheme/data")}, bfs.Data.Dirnames) + + assert.True(bfs.IsData(filepath.Join(workingDir, "mydata", "file1.txt"))) + assert.True(bfs.IsI18n(filepath.Join(workingDir, "myi18n", "file1.txt"))) + assert.True(bfs.IsLayout(filepath.Join(workingDir, "mylayouts", "file1.txt"))) + assert.True(bfs.IsStatic(filepath.Join(workingDir, "mystatic", "file1.txt"))) + contentFilename := filepath.Join(workingDir, "mycontent", "file1.txt") + assert.True(bfs.IsContent(contentFilename)) + rel, _ := bfs.RelContentDir(contentFilename) + assert.Equal("file1.txt", rel) + +} + +func TestNewBaseFsEmpty(t *testing.T) { + assert := require.New(t) + v := viper.New() + v.Set("contentDir", "mycontent") + v.Set("i18nDir", "myi18n") + v.Set("staticDir", "mystatic") + v.Set("dataDir", "mydata") + v.Set("layoutDir", "mylayouts") + v.Set("archetypeDir", "myarchetypes") + + fs := hugofs.NewMem(v) + p, err := paths.New(fs, v) + bfs, err := NewBase(p) + assert.NoError(err) + assert.NotNil(bfs) + assert.Equal(hugofs.NoOpFs, bfs.Archetypes.Fs) + assert.Equal(hugofs.NoOpFs, bfs.Layouts.Fs) + assert.Equal(hugofs.NoOpFs, bfs.Data.Fs) + assert.Equal(hugofs.NoOpFs, bfs.I18n.Fs) + assert.NotNil(hugofs.NoOpFs, bfs.ContentFs) + assert.NotNil(hugofs.NoOpFs, bfs.Static) +} + +func checkFileCount(fs afero.Fs, dirname string, assert *require.Assertions, expected int) { + count, _, err := countFileaAndGetDirs(fs, dirname) + assert.NoError(err) + assert.Equal(expected, count) +} + +func countFileaAndGetDirs(fs afero.Fs, dirname string) (int, []string, error) { + if fs == nil { + return 0, nil, errors.New("no fs") + } + + counter := 0 + var dirs []string + + afero.Walk(fs, dirname, func(path string, info os.FileInfo, err error) error { + if info != nil { + if !info.IsDir() { + counter++ + } else if info.Name() != "." { + dirs = append(dirs, filepath.Join(path, info.Name())) + } + } + + return nil + }) + + return counter, dirs, nil +} + +func setConfigAndWriteSomeFilesTo(fs afero.Fs, v *viper.Viper, key, val string, num int) { + workingDir := v.GetString("workingDir") + v.Set(key, val) + fs.Mkdir(val, 0755) + for i := 0; i < num; i++ { + afero.WriteFile(fs, filepath.Join(workingDir, val, fmt.Sprintf("file%d.txt", i+1)), []byte(fmt.Sprintf("content:%s:%d", key, i+1)), 0755) + } +} diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index ad233f1c2..a0ac72d67 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -25,6 +25,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/i18n" "github.com/gohugoio/hugo/tpl" @@ -228,10 +229,7 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateHandler) error) func(templ tpl.TemplateHandler) error { return func(templ tpl.TemplateHandler) error { - templ.LoadTemplates(s.PathSpec.GetLayoutDirPath(), "") - if s.PathSpec.ThemeSet() { - templ.LoadTemplates(s.PathSpec.GetThemeDir()+"/layouts", "theme") - } + templ.LoadTemplates("") for _, wt := range withTemplates { if wt == nil { @@ -289,7 +287,7 @@ func (h *HugoSites) resetLogs() { } func (h *HugoSites) createSitesFromConfig() error { - oldLangs, _ := h.Cfg.Get("languagesSorted").(helpers.Languages) + oldLangs, _ := h.Cfg.Get("languagesSorted").(langs.Languages) if err := loadLanguageSettings(h.Cfg, oldLangs); err != nil { return err diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 4192580bd..221987b37 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -3,7 +3,6 @@ package hugolib import ( "bytes" "fmt" - "io" "strings" "testing" @@ -12,6 +11,8 @@ import ( "path/filepath" "time" + "github.com/gohugoio/hugo/langs" + "github.com/fortytw2/leaktest" "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/helpers" @@ -660,7 +661,7 @@ title = "Svenska" sites := b.H // Watching does not work with in-memory fs, so we trigger a reload manually - assert.NoError(sites.Cfg.(*helpers.Language).Cfg.(*viper.Viper).ReadInConfig()) + assert.NoError(sites.Cfg.(*langs.Language).Cfg.(*viper.Viper).ReadInConfig()) err := b.H.Build(BuildCfg{CreateSitesFromConfig: true}) if err != nil { @@ -723,7 +724,7 @@ func TestChangeDefaultLanguage(t *testing.T) { // Watching does not work with in-memory fs, so we trigger a reload manually // This does not look pretty, so we should think of something else. - assert.NoError(b.H.Cfg.(*helpers.Language).Cfg.(*viper.Viper).ReadInConfig()) + assert.NoError(b.H.Cfg.(*langs.Language).Cfg.(*viper.Viper).ReadInConfig()) err := b.H.Build(BuildCfg{CreateSitesFromConfig: true}) if err != nil { t.Fatalf("Failed to rebuild sites: %s", err) @@ -1177,31 +1178,12 @@ func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { if err != nil { // Print some debug info root := strings.Split(filename, helpers.FilePathSeparator)[0] - printFs(fs, root, os.Stdout) + helpers.PrintFs(fs, root, os.Stdout) Fatalf(t, "Failed to read file: %s", err) } return string(b) } -func printFs(fs afero.Fs, path string, w io.Writer) { - if fs == nil { - return - } - afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { - if info != nil && !info.IsDir() { - s := path - if lang, ok := info.(hugofs.LanguageAnnouncer); ok { - s = s + "\tLANG: " + lang.Lang() - } - if fp, ok := info.(hugofs.FilePather); ok { - s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir() - } - fmt.Fprintln(w, " ", s) - } - return nil - }) -} - const testPageTemplate = `--- title: "%s" publishdate: "%s" diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go index 7dc2d8e1c..2ccbb6ca1 100644 --- a/hugolib/hugo_sites_multihost_test.go +++ b/hugolib/hugo_sites_multihost_test.go @@ -55,8 +55,6 @@ languageName = "Nynorsk" s1 := b.H.Sites[0] - assert.Equal([]string{"s1", "s2", "ens1", "ens2"}, s1.StaticDirs()) - s1h := s1.getPage(KindHome) assert.True(s1h.IsTranslated()) assert.Len(s1h.Translations(), 2) @@ -79,7 +77,6 @@ languageName = "Nynorsk" b.AssertFileContent("public/en/al/alias2/index.html", `content="0; url=https://example.com/docs/superbob/"`) s2 := b.H.Sites[1] - assert.Equal([]string{"s1", "s2", "frs1", "frs2"}, s2.StaticDirs()) s2h := s2.getPage(KindHome) assert.Equal("https://example.fr/", s2h.Permalink()) diff --git a/hugolib/hugo_themes_test.go b/hugolib/hugo_themes_test.go new file mode 100644 index 000000000..05bfaa692 --- /dev/null +++ b/hugolib/hugo_themes_test.go @@ -0,0 +1,268 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/gohugoio/hugo/common/loggers" +) + +func TestThemesGraph(t *testing.T) { + t.Parallel() + + const ( + themeStandalone = ` +title = "Theme Standalone" +[params] +v1 = "v1s" +v2 = "v2s" +` + themeCyclic = ` +title = "Theme Cyclic" +theme = "theme3" +[params] +v1 = "v1c" +v2 = "v2c" +` + theme1 = ` +title = "Theme #1" +theme = "themeStandalone" +[params] +v2 = "v21" +` + + theme2 = ` +title = "Theme #2" +theme = "theme1" +[params] +v1 = "v12" +` + + theme3 = ` +title = "Theme #3" +theme = ["theme2", "themeStandalone", "themeCyclic"] +[params] +v1 = "v13" +v2 = "v24" +` + + theme4 = ` +title = "Theme #4" +theme = "theme3" +[params] +v1 = "v14" +v2 = "v24" +` + + site1 = ` + theme = "theme4" + + [params] + v1 = "site" +` + site2 = ` + theme = ["theme2", "themeStandalone"] +` + ) + + var ( + testConfigs = []struct { + siteConfig string + + // The name of theme somewhere in the middle to write custom key/files. + offset string + + check func(b *sitesBuilder) + }{ + {site1, "theme3", func(b *sitesBuilder) { + + // site1: theme4 theme3 theme2 theme1 themeStandalone themeCyclic + + // Check data + // theme3 should win the offset competition + b.AssertFileContent("public/index.html", "theme1o::[offset][v]theme3", "theme4o::[offset][v]theme3", "themeStandaloneo::[offset][v]theme3") + b.AssertFileContent("public/index.html", "nproject::[inner][other]project|[project][other]project|[theme][other]theme4|[theme1][other]theme1") + b.AssertFileContent("public/index.html", "ntheme::[inner][other]theme4|[theme][other]theme4|[theme1][other]theme1|[theme2][other]theme2|[theme3][other]theme3") + b.AssertFileContent("public/index.html", "theme1::[inner][other]project|[project][other]project|[theme][other]theme1|[theme1][other]theme1|") + b.AssertFileContent("public/index.html", "theme4::[inner][other]project|[project][other]project|[theme][other]theme4|[theme4][other]theme4|") + + // Check layouts + b.AssertFileContent("public/index.html", "partial ntheme: theme4", "partial theme2o: theme3") + + // Check i18n + b.AssertFileContent("public/index.html", "i18n: project theme4") + + // Check static files + // TODO(bep) static files not currently part of the build b.AssertFileContent("public/nproject.txt", "TODO") + + // Check site params + b.AssertFileContent("public/index.html", "v1::site", "v2::v24") + }}, + {site2, "", func(b *sitesBuilder) { + + // site2: theme2 theme1 themeStandalone + b.AssertFileContent("public/index.html", "nproject::[inner][other]project|[project][other]project|[theme][other]theme2|[theme1][other]theme1|[theme2][other]theme2|[themeStandalone][other]themeStandalone|") + b.AssertFileContent("public/index.html", "ntheme::[inner][other]theme2|[theme][other]theme2|[theme1][other]theme1|[theme2][other]theme2|[themeStandalone][other]themeStandalone|") + b.AssertFileContent("public/index.html", "i18n: project theme2") + b.AssertFileContent("public/index.html", "partial ntheme: theme2") + + // Params only set in themes + b.AssertFileContent("public/index.html", "v1::v12", "v2::v21") + + }}, + } + + themeConfigs = []struct { + name string + config string + }{ + {"themeStandalone", themeStandalone}, + {"themeCyclic", themeCyclic}, + {"theme1", theme1}, + {"theme2", theme2}, + {"theme3", theme3}, + {"theme4", theme4}, + } + ) + + for i, testConfig := range testConfigs { + t.Log(fmt.Sprintf("Test %d", i)) + b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) + b.WithConfigFile("toml", testConfig.siteConfig) + + for _, tc := range themeConfigs { + var variationsNameBase = []string{"nproject", "ntheme", tc.name} + + themeRoot := filepath.Join("themes", tc.name) + b.WithSourceFile(filepath.Join(themeRoot, "config.toml"), tc.config) + + b.WithSourceFile(filepath.Join("layouts", "partials", "m.html"), `{{- range $k, $v := . }}{{ $k }}::{{ template "printv" $v }} +{{ end }} +{{ define "printv" }} +{{- $tp := printf "%T" . -}} +{{- if (strings.HasSuffix $tp "map[string]interface {}") -}} +{{- range $k, $v := . }}[{{ $k }}]{{ template "printv" $v }}{{ end -}} +{{- else -}} +{{- . }}| +{{- end -}} +{{ end }} +`) + + for _, nameVariaton := range variationsNameBase { + roots := []string{"", themeRoot} + + for _, root := range roots { + name := tc.name + if root == "" { + name = "project" + } + + if nameVariaton == "ntheme" && name == "project" { + continue + } + + // static + b.WithSourceFile(filepath.Join(root, "static", nameVariaton+".txt"), name) + + // layouts + if i == 1 { + b.WithSourceFile(filepath.Join(root, "layouts", "partials", "theme2o.html"), "Not Set") + } + b.WithSourceFile(filepath.Join(root, "layouts", "partials", nameVariaton+".html"), name) + if root != "" && testConfig.offset == tc.name { + for _, tc2 := range themeConfigs { + b.WithSourceFile(filepath.Join(root, "layouts", "partials", tc2.name+"o.html"), name) + } + } + + // i18n + data + + var dataContent string + if root == "" { + dataContent = fmt.Sprintf(` +[%s] +other = %q + +[inner] +other = %q + +`, name, name, name) + } else { + dataContent = fmt.Sprintf(` +[%s] +other = %q + +[inner] +other = %q + +[theme] +other = %q + +`, name, name, name, name) + } + + b.WithSourceFile(filepath.Join(root, "data", nameVariaton+".toml"), dataContent) + b.WithSourceFile(filepath.Join(root, "i18n", "en.toml"), dataContent) + + // If an offset is set, duplicate a data key with a winner in the middle. + if root != "" && testConfig.offset == tc.name { + for _, tc2 := range themeConfigs { + dataContent := fmt.Sprintf(` +[offset] +v = %q +`, tc.name) + b.WithSourceFile(filepath.Join(root, "data", tc2.name+"o.toml"), dataContent) + } + } + } + + } + + } + + for _, themeConfig := range themeConfigs { + b.WithSourceFile(filepath.Join("themes", "config.toml"), themeConfig.config) + } + + b.WithContent(filepath.Join("content", "page.md"), `--- +title: "Page" +--- + +`) + + homeTpl := ` +data: {{ partial "m" .Site.Data }} +i18n: {{ i18n "inner" }} {{ i18n "theme" }} +partial ntheme: {{ partial "ntheme" . }} +partial theme2o: {{ partial "theme2o" . }} +params: {{ partial "m" .Site.Params }} + +` + + b.WithTemplates(filepath.Join("layouts", "home.html"), homeTpl) + + b.Build(BuildCfg{}) + + var _ = os.Stdout + + // printFs(b.H.Deps.BaseFs.LayoutsFs, "", os.Stdout) + testConfig.check(b) + + } + +} diff --git a/hugolib/multilingual.go b/hugolib/multilingual.go index a3f3828ef..c09e3667e 100644 --- a/hugolib/multilingual.go +++ b/hugolib/multilingual.go @@ -16,30 +16,33 @@ package hugolib import ( "sync" + "github.com/gohugoio/hugo/common/maps" + "sort" "errors" "fmt" + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" "github.com/spf13/cast" ) // Multilingual manages the all languages used in a multilingual site. type Multilingual struct { - Languages helpers.Languages + Languages langs.Languages - DefaultLang *helpers.Language + DefaultLang *langs.Language - langMap map[string]*helpers.Language + langMap map[string]*langs.Language langMapInit sync.Once } // Language returns the Language associated with the given string. -func (ml *Multilingual) Language(lang string) *helpers.Language { +func (ml *Multilingual) Language(lang string) *langs.Language { ml.langMapInit.Do(func() { - ml.langMap = make(map[string]*helpers.Language) + ml.langMap = make(map[string]*langs.Language) for _, l := range ml.Languages { ml.langMap[l.Lang] = l } @@ -47,16 +50,16 @@ func (ml *Multilingual) Language(lang string) *helpers.Language { return ml.langMap[lang] } -func getLanguages(cfg config.Provider) helpers.Languages { +func getLanguages(cfg config.Provider) langs.Languages { if cfg.IsSet("languagesSorted") { - return cfg.Get("languagesSorted").(helpers.Languages) + return cfg.Get("languagesSorted").(langs.Languages) } - return helpers.Languages{helpers.NewDefaultLanguage(cfg)} + return langs.Languages{langs.NewDefaultLanguage(cfg)} } func newMultiLingualFromSites(cfg config.Provider, sites ...*Site) (*Multilingual, error) { - languages := make(helpers.Languages, len(sites)) + languages := make(langs.Languages, len(sites)) for i, s := range sites { if s.Language == nil { @@ -71,12 +74,12 @@ func newMultiLingualFromSites(cfg config.Provider, sites ...*Site) (*Multilingua defaultLang = "en" } - return &Multilingual{Languages: languages, DefaultLang: helpers.NewLanguage(defaultLang, cfg)}, nil + return &Multilingual{Languages: languages, DefaultLang: langs.NewLanguage(defaultLang, cfg)}, nil } -func newMultiLingualForLanguage(language *helpers.Language) *Multilingual { - languages := helpers.Languages{language} +func newMultiLingualForLanguage(language *langs.Language) *Multilingual { + languages := langs.Languages{language} return &Multilingual{Languages: languages, DefaultLang: language} } func (ml *Multilingual) enabled() bool { @@ -90,8 +93,8 @@ func (s *Site) multilingualEnabled() bool { return s.owner.multilingual != nil && s.owner.multilingual.enabled() } -func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.Languages, error) { - langs := make(helpers.Languages, len(l)) +func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (langs.Languages, error) { + languages := make(langs.Languages, len(l)) i := 0 for lang, langConf := range l { @@ -101,7 +104,7 @@ func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.L return nil, fmt.Errorf("Language config is not a map: %T", langConf) } - language := helpers.NewLanguage(lang, cfg) + language := langs.NewLanguage(lang, cfg) for loki, v := range langsMap { switch loki { @@ -118,7 +121,7 @@ func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.L case "params": m := cast.ToStringMap(v) // Needed for case insensitive fetching of params values - helpers.ToLowerMap(m) + maps.ToLower(m) for k, vv := range m { language.SetParam(k, vv) } @@ -131,11 +134,11 @@ func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.L language.Set(loki, v) } - langs[i] = language + languages[i] = language i++ } - sort.Sort(langs) + sort.Sort(languages) - return langs, nil + return languages, nil } diff --git a/hugolib/page.go b/hugolib/page.go index 89d68084e..322660647 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -21,6 +21,10 @@ import ( "reflect" "unicode" + "github.com/gohugoio/hugo/common/maps" + + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/related" "github.com/bep/gitmap" @@ -254,7 +258,7 @@ type Page struct { // It would be tempting to use the language set on the Site, but in they way we do // multi-site processing, these values may differ during the initial page processing. - language *helpers.Language + language *langs.Language lang string @@ -1281,7 +1285,7 @@ func (p *Page) update(frontmatter map[string]interface{}) error { return errors.New("missing frontmatter data") } // Needed for case insensitive fetching of params values - helpers.ToLowerMap(frontmatter) + maps.ToLower(frontmatter) var mtime time.Time if p.Source.FileInfo() != nil { @@ -2028,7 +2032,7 @@ func (p *Page) Scratch() *Scratch { return p.scratch } -func (p *Page) Language() *helpers.Language { +func (p *Page) Language() *langs.Language { p.initLanguage() return p.language } diff --git a/hugolib/page_bundler_capture.go b/hugolib/page_bundler_capture.go index 255a8efda..92b3efe49 100644 --- a/hugolib/page_bundler_capture.go +++ b/hugolib/page_bundler_capture.go @@ -75,7 +75,7 @@ func newCapturer( sem: make(chan bool, numWorkers), handler: handler, sourceSpec: sourceSpec, - fs: sourceSpec.Fs, + fs: sourceSpec.SourceFs, logger: logger, contentChanges: contentChanges, seen: make(map[string]bool), diff --git a/hugolib/page_bundler_capture_test.go b/hugolib/page_bundler_capture_test.go index c07383797..14d8a4368 100644 --- a/hugolib/page_bundler_capture_test.go +++ b/hugolib/page_bundler_capture_test.go @@ -20,6 +20,8 @@ import ( "path/filepath" "sort" + "github.com/gohugoio/hugo/common/loggers" + jww "github.com/spf13/jwalterweatherman" "runtime" @@ -92,7 +94,7 @@ func TestPageBundlerCaptureSymlinks(t *testing.T) { sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) fileStore := &storeFilenames{} - logger := newErrorLogger() + logger := loggers.NewErrorLogger() c := newCapturer(logger, sourceSpec, fileStore, nil) assert.NoError(c.capture()) @@ -139,12 +141,10 @@ func TestPageBundlerCaptureBasic(t *testing.T) { fileStore := &storeFilenames{} - c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil) + c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) assert.NoError(c.capture()) - printFs(fs.Source, "", os.Stdout) - expected := ` F: /work/base/_1.md @@ -185,7 +185,7 @@ func TestPageBundlerCaptureMultilingual(t *testing.T) { sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) fileStore := &storeFilenames{} - c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil) + c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) assert.NoError(c.capture()) @@ -265,7 +265,7 @@ func BenchmarkPageBundlerCapture(b *testing.B) { writeSource(b, fs, filepath.Join(base, "contentonly", fmt.Sprintf("c%d.md", i)), "content") } - capturers[i] = newCapturer(newErrorLogger(), sourceSpec, new(noOpFileStore), nil, base) + capturers[i] = newCapturer(loggers.NewErrorLogger(), sourceSpec, new(noOpFileStore), nil, base) } b.ResetTimer() diff --git a/hugolib/page_bundler_test.go b/hugolib/page_bundler_test.go index 2d83e3af1..f66b8c0db 100644 --- a/hugolib/page_bundler_test.go +++ b/hugolib/page_bundler_test.go @@ -15,6 +15,9 @@ package hugolib import ( "io/ioutil" + + "github.com/gohugoio/hugo/common/loggers" + "os" "runtime" "strings" @@ -75,7 +78,7 @@ func TestPageBundlerSiteRegular(t *testing.T) { cfg.Set("uglyURLs", ugly) - s := buildSingleSite(t, deps.DepsCfg{Logger: newWarningLogger(), Fs: fs, Cfg: cfg}, BuildCfg{}) + s := buildSingleSite(t, deps.DepsCfg{Logger: loggers.NewWarningLogger(), Fs: fs, Cfg: cfg}, BuildCfg{}) th := testHelper{s.Cfg, s.Fs, t} @@ -158,7 +161,6 @@ func TestPageBundlerSiteRegular(t *testing.T) { assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/c/logo.png"), image.(resource.Source).AbsSourceFilename()) assert.Equal("https://example.com/2017/pageslug/c/logo.png", image.Permalink()) - printFs(th.Fs.Destination, "", os.Stdout) th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content") @@ -329,7 +331,7 @@ func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) { cfg := ps.Cfg fs := ps.Fs - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: newErrorLogger()}, BuildCfg{}) + s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: loggers.NewErrorLogger()}, BuildCfg{}) th := testHelper{s.Cfg, s.Fs, t} diff --git a/hugolib/pagination.go b/hugolib/pagination.go index 86113271b..84ad74b07 100644 --- a/hugolib/pagination.go +++ b/hugolib/pagination.go @@ -532,7 +532,7 @@ func newPaginationURLFactory(d targetPathDescriptor) paginationURLFactory { pathDescriptor := d var rel string if page > 1 { - rel = fmt.Sprintf("/%s/%d/", d.PathSpec.PaginatePath(), page) + rel = fmt.Sprintf("/%s/%d/", d.PathSpec.PaginatePath, page) pathDescriptor.Addends = rel } diff --git a/helpers/baseURL.go b/hugolib/paths/baseURL.go index eb39ced5b..9cb5627ba 100644 --- a/helpers/baseURL.go +++ b/hugolib/paths/baseURL.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package paths import ( "fmt" @@ -30,6 +30,10 @@ func (b BaseURL) String() string { return b.urlStr } +func (b BaseURL) Path() string { + return b.url.Path +} + // WithProtocol returns the BaseURL prefixed with the given protocol. // The Protocol is normally of the form "scheme://", i.e. "webcal://". func (b BaseURL) WithProtocol(protocol string) (string, error) { diff --git a/helpers/baseURL_test.go b/hugolib/paths/baseURL_test.go index 437152f34..af1d2e38d 100644 --- a/helpers/baseURL_test.go +++ b/hugolib/paths/baseURL_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package paths import ( "testing" diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go new file mode 100644 index 000000000..cf8792e5a --- /dev/null +++ b/hugolib/paths/paths.go @@ -0,0 +1,231 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paths + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/langs" + + "github.com/gohugoio/hugo/hugofs" +) + +var FilePathSeparator = string(filepath.Separator) + +type Paths struct { + Fs *hugofs.Fs + Cfg config.Provider + + BaseURL + + // If the baseURL contains a base path, e.g. https://example.com/docs, then "/docs" will be the BasePath. + // This will not be set if canonifyURLs is enabled. + BasePath string + + // Directories + // TODO(bep) when we have trimmed down mos of the dirs usage outside of this package, make + // these into an interface. + ContentDir string + ThemesDir string + WorkingDir string + AbsResourcesDir string + AbsPublishDir string + + // pagination path handling + PaginatePath string + + PublishDir string + + DisablePathToLower bool + RemovePathAccents bool + UglyURLs bool + CanonifyURLs bool + + Language *langs.Language + Languages langs.Languages + + // The PathSpec looks up its config settings in both the current language + // and then in the global Viper config. + // Some settings, the settings listed below, does not make sense to be set + // on per-language-basis. We have no good way of protecting against this + // other than a "white-list". See language.go. + defaultContentLanguageInSubdir bool + DefaultContentLanguage string + multilingual bool + + themes []string + AllThemes []ThemeConfig +} + +func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { + baseURLstr := cfg.GetString("baseURL") + baseURL, err := newBaseURLFromString(baseURLstr) + + if err != nil { + return nil, fmt.Errorf("Failed to create baseURL from %q: %s", baseURLstr, err) + } + + // TODO(bep) + contentDir := cfg.GetString("contentDir") + workingDir := cfg.GetString("workingDir") + resourceDir := cfg.GetString("resourceDir") + publishDir := cfg.GetString("publishDir") + + defaultContentLanguage := cfg.GetString("defaultContentLanguage") + + var ( + language *langs.Language + languages langs.Languages + ) + + if l, ok := cfg.(*langs.Language); ok { + language = l + + } + + if l, ok := cfg.Get("languagesSorted").(langs.Languages); ok { + languages = l + } + + if len(languages) == 0 { + // We have some old tests that does not test the entire chain, hence + // they have no languages. So create one so we get the proper filesystem. + languages = langs.Languages{&langs.Language{Lang: "en", Cfg: cfg, ContentDir: contentDir}} + } + + absPublishDir := AbsPathify(workingDir, publishDir) + if !strings.HasSuffix(absPublishDir, FilePathSeparator) { + absPublishDir += FilePathSeparator + } + // If root, remove the second '/' + if absPublishDir == "//" { + absPublishDir = FilePathSeparator + } + absResourcesDir := AbsPathify(workingDir, resourceDir) + if !strings.HasSuffix(absResourcesDir, FilePathSeparator) { + absResourcesDir += FilePathSeparator + } + if absResourcesDir == "//" { + absResourcesDir = FilePathSeparator + } + + p := &Paths{ + Fs: fs, + Cfg: cfg, + BaseURL: baseURL, + + DisablePathToLower: cfg.GetBool("disablePathToLower"), + RemovePathAccents: cfg.GetBool("removePathAccents"), + UglyURLs: cfg.GetBool("uglyURLs"), + CanonifyURLs: cfg.GetBool("canonifyURLs"), + + ContentDir: contentDir, + ThemesDir: cfg.GetString("themesDir"), + WorkingDir: workingDir, + + AbsResourcesDir: absResourcesDir, + AbsPublishDir: absPublishDir, + + themes: config.GetStringSlicePreserveString(cfg, "theme"), + + multilingual: cfg.GetBool("multilingual"), + defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"), + DefaultContentLanguage: defaultContentLanguage, + + Language: language, + Languages: languages, + + PaginatePath: cfg.GetString("paginatePath"), + } + + if cfg.IsSet("allThemes") { + p.AllThemes = cfg.Get("allThemes").([]ThemeConfig) + } else { + p.AllThemes, err = collectThemeNames(p) + if err != nil { + return nil, err + } + } + + // TODO(bep) remove this, eventually + p.PublishDir = absPublishDir + + return p, nil +} + +func (p *Paths) Lang() string { + if p == nil || p.Language == nil { + return "" + } + return p.Language.Lang +} + +// ThemeSet checks whether a theme is in use or not. +func (p *Paths) ThemeSet() bool { + return len(p.themes) > 0 +} + +func (p *Paths) Themes() []string { + return p.themes +} + +func (p *Paths) GetLanguagePrefix() string { + if !p.multilingual { + return "" + } + + defaultLang := p.DefaultContentLanguage + defaultInSubDir := p.defaultContentLanguageInSubdir + + currentLang := p.Language.Lang + if currentLang == "" || (currentLang == defaultLang && !defaultInSubDir) { + return "" + } + return currentLang +} + +// GetLangSubDir returns the given language's subdir if needed. +func (p *Paths) GetLangSubDir(lang string) string { + if !p.multilingual { + return "" + } + + if p.Languages.IsMultihost() { + return "" + } + + if lang == "" || (lang == p.DefaultContentLanguage && !p.defaultContentLanguageInSubdir) { + return "" + } + + return lang +} + +// AbsPathify creates an absolute path if given a relative path. If already +// absolute, the path is just cleaned. +func (p *Paths) AbsPathify(inPath string) string { + return AbsPathify(p.WorkingDir, inPath) +} + +// AbsPathify creates an absolute path if given a working dir and arelative path. +// If already absolute, the path is just cleaned. +func AbsPathify(workingDir, inPath string) string { + if filepath.IsAbs(inPath) { + return filepath.Clean(inPath) + } + return filepath.Join(workingDir, inPath) +} diff --git a/hugolib/paths/paths_test.go b/hugolib/paths/paths_test.go new file mode 100644 index 000000000..6cadc747f --- /dev/null +++ b/hugolib/paths/paths_test.go @@ -0,0 +1,40 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paths + +import ( + "testing" + + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestNewPaths(t *testing.T) { + assert := require.New(t) + + v := viper.New() + fs := hugofs.NewMem(v) + + v.Set("defaultContentLanguageInSubdir", true) + v.Set("defaultContentLanguage", "no") + v.Set("multilingual", true) + + p, err := New(fs, v) + assert.NoError(err) + + assert.Equal(true, p.defaultContentLanguageInSubdir) + assert.Equal("no", p.DefaultContentLanguage) + assert.Equal(true, p.multilingual) +} diff --git a/hugolib/paths/themes.go b/hugolib/paths/themes.go new file mode 100644 index 000000000..abe6121ae --- /dev/null +++ b/hugolib/paths/themes.go @@ -0,0 +1,162 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paths + +import ( + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" + "github.com/spf13/cast" + "github.com/spf13/viper" +) + +type ThemeConfig struct { + // The theme name as provided by the folder name below /themes. + Name string + + // Optional configuration filename (e.g. "/themes/mytheme/config.json"). + ConfigFilename string + + // Optional config read from the ConfigFile above. + Cfg config.Provider +} + +// Create file system, an ordered theme list from left to right, no duplicates. +type themesCollector struct { + themesDir string + fs afero.Fs + seen map[string]bool + themes []ThemeConfig +} + +func (c *themesCollector) isSeen(theme string) bool { + loki := strings.ToLower(theme) + if c.seen[loki] { + return true + } + c.seen[loki] = true + return false +} + +func (c *themesCollector) addAndRecurse(themes ...string) error { + for i := 0; i < len(themes); i++ { + theme := themes[i] + configFilename := c.getConfigFileIfProvided(theme) + if !c.isSeen(theme) { + tc, err := c.add(theme, configFilename) + if err != nil { + return err + } + if err := c.addTemeNamesFromTheme(tc); err != nil { + return err + } + } + } + return nil +} + +func (c *themesCollector) add(name, configFilename string) (ThemeConfig, error) { + var cfg config.Provider + var tc ThemeConfig + + if configFilename != "" { + v := viper.New() + v.SetFs(c.fs) + v.AutomaticEnv() + v.SetEnvPrefix("hugo") + v.SetConfigFile(configFilename) + + err := v.ReadInConfig() + if err != nil { + return tc, err + } + cfg = v + + } + + tc = ThemeConfig{Name: name, ConfigFilename: configFilename, Cfg: cfg} + c.themes = append(c.themes, tc) + return tc, nil + +} + +func collectThemeNames(p *Paths) ([]ThemeConfig, error) { + return CollectThemes(p.Fs.Source, p.AbsPathify(p.ThemesDir), p.Themes()) + +} + +func CollectThemes(fs afero.Fs, themesDir string, themes []string) ([]ThemeConfig, error) { + if len(themes) == 0 { + return nil, nil + } + + c := &themesCollector{ + fs: fs, + themesDir: themesDir, + seen: make(map[string]bool)} + + for i := 0; i < len(themes); i++ { + theme := themes[i] + if err := c.addAndRecurse(theme); err != nil { + return nil, err + } + } + + return c.themes, nil + +} + +func (c *themesCollector) getConfigFileIfProvided(theme string) string { + configDir := filepath.Join(c.themesDir, theme) + + var ( + configFilename string + exists bool + ) + + // Viper supports more, but this is the sub-set supported by Hugo. + for _, configFormats := range []string{"toml", "yaml", "yml", "json"} { + configFilename = filepath.Join(configDir, "config."+configFormats) + exists, _ = afero.Exists(c.fs, configFilename) + if exists { + break + } + } + + if !exists { + // No theme config set. + return "" + } + + return configFilename + +} + +func (c *themesCollector) addTemeNamesFromTheme(theme ThemeConfig) error { + if theme.Cfg != nil && theme.Cfg.IsSet("theme") { + v := theme.Cfg.Get("theme") + switch vv := v.(type) { + case []string: + return c.addAndRecurse(vv...) + case []interface{}: + return c.addAndRecurse(cast.ToStringSlice(vv)...) + default: + return c.addAndRecurse(cast.ToString(vv)) + } + } + + return nil +} diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index a4c6ca20d..1437ae0cf 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -34,7 +34,9 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/tpl" + "github.com/stretchr/testify/require" ) @@ -46,7 +48,7 @@ func pageFromString(in, filename string, withTemplate ...func(templ tpl.Template var err error cfg, fs := newTestCfg() - d := deps.DepsCfg{Language: helpers.NewLanguage("en", cfg), Cfg: cfg, Fs: fs, WithTemplate: withTemplate[0]} + d := deps.DepsCfg{Language: langs.NewLanguage("en", cfg), Cfg: cfg, Fs: fs, WithTemplate: withTemplate[0]} s, err = NewSiteForCfg(d) if err != nil { diff --git a/hugolib/site.go b/hugolib/site.go index 04a18bb48..8ff724d0a 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -27,6 +27,10 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/langs" + + src "github.com/gohugoio/hugo/source" + "github.com/gohugoio/hugo/resource" "golang.org/x/sync/errgroup" @@ -107,7 +111,7 @@ type Site struct { expiredCount int Data map[string]interface{} - Language *helpers.Language + Language *langs.Language disabledKinds map[string]bool @@ -175,7 +179,7 @@ func (s *Site) isEnabled(kind string) bool { // reset returns a new Site prepared for rebuild. func (s *Site) reset() *Site { return &Site{Deps: s.Deps, - layoutHandler: output.NewLayoutHandler(s.PathSpec.ThemeSet()), + layoutHandler: output.NewLayoutHandler(), disabledKinds: s.disabledKinds, titleFunc: s.titleFunc, relatedDocsHandler: newSearchIndexHandler(s.relatedDocsHandler.cfg), @@ -195,7 +199,7 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { c := newPageCollections() if cfg.Language == nil { - cfg.Language = helpers.NewDefaultLanguage(cfg.Cfg) + cfg.Language = langs.NewDefaultLanguage(cfg.Cfg) } disabledKinds := make(map[string]bool) @@ -261,7 +265,7 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { s := &Site{ PageCollections: c, - layoutHandler: output.NewLayoutHandler(cfg.Cfg.GetString("themesDir") != ""), + layoutHandler: output.NewLayoutHandler(), Language: cfg.Language, disabledKinds: disabledKinds, titleFunc: titleFunc, @@ -304,7 +308,7 @@ func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateHandler) error) ( if err := loadDefaultSettingsFor(v); err != nil { return nil, err } - return newSiteForLang(helpers.NewDefaultLanguage(v), withTemplate...) + return newSiteForLang(langs.NewDefaultLanguage(v), withTemplate...) } // NewEnglishSite creates a new site in English language. @@ -316,11 +320,11 @@ func NewEnglishSite(withTemplate ...func(templ tpl.TemplateHandler) error) (*Sit if err := loadDefaultSettingsFor(v); err != nil { return nil, err } - return newSiteForLang(helpers.NewLanguage("en", v), withTemplate...) + return newSiteForLang(langs.NewLanguage("en", v), withTemplate...) } // newSiteForLang creates a new site in the given language. -func newSiteForLang(lang *helpers.Language, withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) { +func newSiteForLang(lang *langs.Language, withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) { withTemplates := func(templ tpl.TemplateHandler) error { for _, wt := range withTemplate { if err := wt(templ); err != nil { @@ -389,9 +393,9 @@ type SiteInfo struct { owner *HugoSites s *Site multilingual *Multilingual - Language *helpers.Language + Language *langs.Language LanguagePrefix string - Languages helpers.Languages + Languages langs.Languages defaultContentLanguageInSubdir bool sectionPagesMenu string } @@ -431,7 +435,7 @@ func (s *SiteInfo) DisqusShortname() string { // Used in tests. type siteBuilderCfg struct { - language *helpers.Language + language *langs.Language s *Site pageCollections *PageCollections } @@ -805,15 +809,13 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { } -func (s *Site) loadData(sourceDirs []string) (err error) { - s.Log.DEBUG.Printf("Load Data from %d source(s)", len(sourceDirs)) +func (s *Site) loadData(fs afero.Fs) (err error) { + spec := src.NewSourceSpec(s.PathSpec, fs) + fileSystem := spec.NewFilesystem("") s.Data = make(map[string]interface{}) - for _, sourceDir := range sourceDirs { - fs := s.SourceSpec.NewFilesystem(sourceDir) - for _, r := range fs.Files() { - if err := s.handleDataFile(r); err != nil { - return err - } + for _, r := range fileSystem.Files() { + if err := s.handleDataFile(r); err != nil { + return err } } @@ -831,12 +833,17 @@ func (s *Site) handleDataFile(r source.ReadableFile) error { // Crawl in data tree to insert data current = s.Data - for _, key := range strings.Split(r.Dir(), helpers.FilePathSeparator) { - if key != "" { - if _, ok := current[key]; !ok { - current[key] = make(map[string]interface{}) + keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator) + // The first path element is the virtual folder (typically theme name), which is + // not part of the key. + if len(keyParts) > 1 { + for _, key := range keyParts[1:] { + if key != "" { + if _, ok := current[key]; !ok { + current[key] = make(map[string]interface{}) + } + current = current[key].(map[string]interface{}) } - current = current[key].(map[string]interface{}) } } @@ -919,18 +926,7 @@ func (s *Site) readData(f source.ReadableFile) (interface{}, error) { } func (s *Site) readDataFromSourceFS() error { - var dataSourceDirs []string - - // have to be last - duplicate keys in earlier entries will win - themeDataDir, err := s.PathSpec.GetThemeDataDirPath() - if err == nil { - dataSourceDirs = []string{s.absDataDir(), themeDataDir} - } else { - dataSourceDirs = []string{s.absDataDir()} - - } - - err = s.loadData(dataSourceDirs) + err := s.loadData(s.PathSpec.BaseFs.Data.Fs) s.timerStep("load data") return err } @@ -1041,10 +1037,6 @@ func (s *Site) Initialise() (err error) { func (s *Site) initialize() (err error) { s.Menus = Menus{} - if err = s.checkDirectories(); err != nil { - return err - } - return s.initializeSiteInfo() } @@ -1071,7 +1063,7 @@ func (s *SiteInfo) SitemapAbsURL() string { func (s *Site) initializeSiteInfo() error { var ( lang = s.Language - languages helpers.Languages + languages langs.Languages ) if s.owner != nil && s.owner.multilingual != nil { @@ -1166,126 +1158,24 @@ func (s *Site) initializeSiteInfo() error { return nil } -func (s *Site) dataDir() string { - return s.Cfg.GetString("dataDir") -} - -func (s *Site) absDataDir() string { - return s.PathSpec.AbsPathify(s.dataDir()) -} - -func (s *Site) i18nDir() string { - return s.Cfg.GetString("i18nDir") -} - -func (s *Site) absI18nDir() string { - return s.PathSpec.AbsPathify(s.i18nDir()) -} - func (s *Site) isI18nEvent(e fsnotify.Event) bool { - if s.getI18nDir(e.Name) != "" { - return true - } - return s.getThemeI18nDir(e.Name) != "" -} - -func (s *Site) getI18nDir(path string) string { - return s.getRealDir(s.absI18nDir(), path) -} - -func (s *Site) getThemeI18nDir(path string) string { - if !s.PathSpec.ThemeSet() { - return "" - } - return s.getRealDir(filepath.Join(s.PathSpec.GetThemeDir(), s.i18nDir()), path) + return s.BaseFs.SourceFilesystems.IsI18n(e.Name) } func (s *Site) isDataDirEvent(e fsnotify.Event) bool { - if s.getDataDir(e.Name) != "" { - return true - } - return s.getThemeDataDir(e.Name) != "" -} - -func (s *Site) getDataDir(path string) string { - return s.getRealDir(s.absDataDir(), path) -} - -func (s *Site) getThemeDataDir(path string) string { - if !s.PathSpec.ThemeSet() { - return "" - } - return s.getRealDir(filepath.Join(s.PathSpec.GetThemeDir(), s.dataDir()), path) -} - -func (s *Site) layoutDir() string { - return s.Cfg.GetString("layoutDir") + return s.BaseFs.SourceFilesystems.IsData(e.Name) } func (s *Site) isLayoutDirEvent(e fsnotify.Event) bool { - if s.getLayoutDir(e.Name) != "" { - return true - } - return s.getThemeLayoutDir(e.Name) != "" -} - -func (s *Site) getLayoutDir(path string) string { - return s.getRealDir(s.PathSpec.GetLayoutDirPath(), path) -} - -func (s *Site) getThemeLayoutDir(path string) string { - if !s.PathSpec.ThemeSet() { - return "" - } - return s.getRealDir(filepath.Join(s.PathSpec.GetThemeDir(), s.layoutDir()), path) + return s.BaseFs.SourceFilesystems.IsLayout(e.Name) } func (s *Site) absContentDir() string { - return s.PathSpec.AbsPathify(s.PathSpec.ContentDir()) + return s.PathSpec.AbsPathify(s.PathSpec.ContentDir) } func (s *Site) isContentDirEvent(e fsnotify.Event) bool { - relDir, _ := s.PathSpec.RelContentDir(e.Name) - return relDir != e.Name -} - -func (s *Site) getContentDir(path string) string { - return s.getRealDir(s.absContentDir(), path) -} - -// getRealDir gets the base path of the given path, also handling the case where -// base is a symlinked folder. -func (s *Site) getRealDir(base, path string) string { - - if strings.HasPrefix(path, base) { - return base - } - - realDir, err := helpers.GetRealPath(s.Fs.Source, base) - - if err != nil { - if !os.IsNotExist(err) { - s.Log.ERROR.Printf("Failed to get real path for %s: %s", path, err) - } - return "" - } - - if strings.HasPrefix(path, realDir) { - return realDir - } - - return "" -} - -func (s *Site) absPublishDir() string { - return s.PathSpec.AbsPathify(s.Cfg.GetString("publishDir")) -} - -func (s *Site) checkDirectories() (err error) { - if b, _ := helpers.DirExists(s.absContentDir(), s.Fs.Source); !b { - return errors.New("No source directory found, expecting to find it at " + s.absContentDir()) - } - return + return s.BaseFs.IsContent(e.Name) } type contentCaptureResultHandler struct { @@ -1871,9 +1761,7 @@ func (s *Site) findFirstTemplate(layouts ...string) tpl.Template { func (s *Site) publish(statCounter *uint64, path string, r io.Reader) (err error) { s.PathSpec.ProcessingStats.Incr(statCounter) - path = filepath.Join(s.absPublishDir(), path) - - return helpers.WriteToDisk(path, r, s.Fs.Destination) + return helpers.WriteToDisk(filepath.Clean(path), r, s.BaseFs.PublishFs) } func getGoMaxProcs() int { diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 231200a7b..93ea5032e 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -10,8 +10,8 @@ import ( "strings" "text/template" + "github.com/gohugoio/hugo/langs" "github.com/sanity-io/litter" - jww "github.com/spf13/jwalterweatherman" "github.com/gohugoio/hugo/config" @@ -22,11 +22,8 @@ import ( "github.com/gohugoio/hugo/tpl" "github.com/spf13/viper" - "io/ioutil" "os" - "log" - "github.com/gohugoio/hugo/hugofs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -135,6 +132,11 @@ func (s *sitesBuilder) WithThemeConfigFile(format, conf string) *sitesBuilder { return s } +func (s *sitesBuilder) WithSourceFile(filename, content string) *sitesBuilder { + writeSource(s.T, s.Fs, filepath.FromSlash(filename), content) + return s +} + const commonConfigSections = ` [services] @@ -304,15 +306,17 @@ func (s *sitesBuilder) CreateSites() *sitesBuilder { s.writeFilePairs("i18n", s.i18nFilePairsAdded) if s.Cfg == nil { - cfg, configFiles, err := LoadConfig(ConfigSourceDescriptor{Fs: s.Fs.Source, Filename: "config." + s.configFormat}) + cfg, _, err := LoadConfig(ConfigSourceDescriptor{Fs: s.Fs.Source, Filename: "config." + s.configFormat}) if err != nil { s.Fatalf("Failed to load config: %s", err) } - expectedConfigs := 1 - if s.theme != "" { - expectedConfigs = 2 - } - require.Equal(s.T, expectedConfigs, len(configFiles), fmt.Sprintf("Configs: %v", configFiles)) + // TODO(bep) + /* expectedConfigs := 1 + if s.theme != "" { + expectedConfigs = 2 + } + require.Equal(s.T, expectedConfigs, len(configFiles), fmt.Sprintf("Configs: %v", configFiles)) + */ s.Cfg = cfg } @@ -337,6 +341,7 @@ func (s *sitesBuilder) build(cfg BuildCfg, shouldFail bool) *sitesBuilder { if s.H == nil { s.CreateSites() } + err := s.H.Build(cfg) if err == nil { logErrorCount := s.H.NumLogErrors() @@ -436,7 +441,7 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { content := readDestination(s.T, s.Fs, filename) for _, match := range matches { if !strings.Contains(content, match) { - s.Fatalf("No match for %q in content for %s\n%q", match, filename, content) + s.Fatalf("No match for %q in content for %s\n%s", match, filename, content) } } } @@ -509,7 +514,7 @@ func (th testHelper) replaceDefaultContentLanguageValue(value string) string { } func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *helpers.PathSpec { - l := helpers.NewDefaultLanguage(v) + l := langs.NewDefaultLanguage(v) ps, _ := helpers.NewPathSpec(fs, l) return ps } @@ -519,6 +524,10 @@ func newTestDefaultPathSpec() *helpers.PathSpec { // Easier to reason about in tests. v.Set("disablePathToLower", true) v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") fs := hugofs.NewDefault(v) ps, _ := helpers.NewPathSpec(fs, v) return ps @@ -551,7 +560,7 @@ func newTestSite(t testing.TB, configKeyValues ...interface{}) *Site { cfg.Set(configKeyValues[i].(string), configKeyValues[i+1]) } - d := deps.DepsCfg{Language: helpers.NewLanguage("en", cfg), Fs: fs, Cfg: cfg} + d := deps.DepsCfg{Language: langs.NewLanguage("en", cfg), Fs: fs, Cfg: cfg} s, err := NewSiteForCfg(d) @@ -593,18 +602,6 @@ func newTestSitesFromConfigWithDefaultTemplates(t testing.TB, tomlConfig string) ) } -func newDebugLogger() *jww.Notepad { - return jww.NewNotepad(jww.LevelDebug, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) -} - -func newErrorLogger() *jww.Notepad { - return jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) -} - -func newWarningLogger() *jww.Notepad { - return jww.NewNotepad(jww.LevelWarn, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) -} - func createWithTemplateFromNameValues(additionalTemplates ...string) func(templ tpl.TemplateHandler) error { return func(templ tpl.TemplateHandler) error { diff --git a/i18n/i18n_test.go b/i18n/i18n_test.go index 4f5b3fbac..c5c962c16 100644 --- a/i18n/i18n_test.go +++ b/i18n/i18n_test.go @@ -19,6 +19,7 @@ import ( "github.com/gohugoio/hugo/tpl/tplimpl" + "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" "github.com/gohugoio/hugo/deps" @@ -26,8 +27,6 @@ import ( "io/ioutil" "os" - "github.com/gohugoio/hugo/helpers" - "log" "github.com/gohugoio/hugo/config" @@ -168,15 +167,16 @@ func doTestI18nTranslate(t *testing.T, test i18nTest, cfg config.Provider) strin assert := require.New(t) fs := hugofs.NewMem(cfg) tp := NewTranslationProvider() - depsCfg := newDepsConfig(tp, cfg, fs) - d, err := deps.New(depsCfg) - assert.NoError(err) for file, content := range test.data { err := afero.WriteFile(fs.Source, filepath.Join("i18n", file), []byte(content), 0755) assert.NoError(err) } + depsCfg := newDepsConfig(tp, cfg, fs) + d, err := deps.New(depsCfg) + assert.NoError(err) + assert.NoError(d.LoadResources()) f := tp.t.Func(test.lang) return f(test.id, test.args) @@ -184,7 +184,7 @@ func doTestI18nTranslate(t *testing.T, test i18nTest, cfg config.Provider) strin } func newDepsConfig(tp *TranslationProvider, cfg config.Provider, fs *hugofs.Fs) deps.DepsCfg { - l := helpers.NewLanguage("en", cfg) + l := langs.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") return deps.DepsCfg{ Language: l, @@ -201,6 +201,10 @@ func TestI18nTranslate(t *testing.T) { v := viper.New() v.SetDefault("defaultContentLanguage", "en") v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") // Test without and with placeholders for _, enablePlaceholders := range []bool{false, true} { diff --git a/i18n/translationProvider.go b/i18n/translationProvider.go index fa5664210..8749360b3 100644 --- a/i18n/translationProvider.go +++ b/i18n/translationProvider.go @@ -38,17 +38,8 @@ func NewTranslationProvider() *TranslationProvider { // Update updates the i18n func in the provided Deps. func (tp *TranslationProvider) Update(d *deps.Deps) error { - dir := d.PathSpec.AbsPathify(d.Cfg.GetString("i18nDir")) - sp := source.NewSourceSpec(d.PathSpec, d.Fs.Source) - sources := []source.Input{sp.NewFilesystem(dir)} - - themeI18nDir, err := d.PathSpec.GetThemeI18nDirPath() - - if err == nil { - sources = []source.Input{sp.NewFilesystem(themeI18nDir), sources[0]} - } - - d.Log.DEBUG.Printf("Load I18n from %q", sources) + sp := source.NewSourceSpec(d.PathSpec, d.BaseFs.SourceFilesystems.I18n.Fs) + src := sp.NewFilesystem("") i18nBundle := bundle.New() @@ -58,14 +49,12 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { } var newLangs []string - for _, currentSource := range sources { - for _, r := range currentSource.Files() { - currentSpec := language.GetPluralSpec(r.BaseFileName()) - if currentSpec == nil { - // This may is a language code not supported by go-i18n, it may be - // Klingon or ... not even a fake language. Make sure it works. - newLangs = append(newLangs, r.BaseFileName()) - } + for _, r := range src.Files() { + currentSpec := language.GetPluralSpec(r.BaseFileName()) + if currentSpec == nil { + // This may is a language code not supported by go-i18n, it may be + // Klingon or ... not even a fake language. Make sure it works. + newLangs = append(newLangs, r.BaseFileName()) } } @@ -73,11 +62,12 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { language.RegisterPluralSpec(newLangs, en) } - for _, currentSource := range sources { - for _, r := range currentSource.Files() { - if err := addTranslationFile(i18nBundle, r); err != nil { - return err - } + // The source files are ordered so the most important comes first. Since this is a + // last key win situation, we have to reverse the iteration order. + files := src.Files() + for i := len(files) - 1; i >= 0; i-- { + if err := addTranslationFile(i18nBundle, files[i]); err != nil { + return err } } diff --git a/helpers/language.go b/langs/language.go index 731e9b088..6f3e1de64 100644 --- a/helpers/language.go +++ b/langs/language.go @@ -1,4 +1,4 @@ -// Copyright 2016-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,12 +11,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package langs import ( "sort" "strings" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/config" "github.com/spf13/cast" ) @@ -73,7 +74,7 @@ func NewLanguage(lang string, cfg config.Provider) *Language { for k, v := range cfg.GetStringMap("params") { params[k] = v } - ToLowerMap(params) + maps.ToLower(params) defaultContentDir := cfg.GetString("contentDir") if defaultContentDir == "" { diff --git a/helpers/language_test.go b/langs/language_test.go index 4c4670321..8783172fb 100644 --- a/helpers/language_test.go +++ b/langs/language_test.go @@ -1,4 +1,4 @@ -// Copyright 2016-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package langs import ( "testing" diff --git a/output/docshelper.go b/output/docshelper.go index 6b7826002..fcf9ae61c 100644 --- a/output/docshelper.go +++ b/output/docshelper.go @@ -68,7 +68,7 @@ func createLayoutExamples() interface{} { {"Taxonomy term in categories", LayoutDescriptor{Kind: "taxonomyTerm", Type: "categories", Section: "category"}, false, HTMLFormat}, } { - l := NewLayoutHandler(example.hasTheme) + l := NewLayoutHandler() layouts, _ := l.For(example.d, example.f) basicExamples = append(basicExamples, Example{ diff --git a/output/layout.go b/output/layout.go index 206293842..f83490d81 100644 --- a/output/layout.go +++ b/output/layout.go @@ -41,8 +41,6 @@ type LayoutDescriptor struct { // LayoutHandler calculates the layout template to use to render a given output type. type LayoutHandler struct { - hasTheme bool - mu sync.RWMutex cache map[layoutCacheKey][]string } @@ -53,8 +51,8 @@ type layoutCacheKey struct { } // NewLayoutHandler creates a new LayoutHandler. -func NewLayoutHandler(hasTheme bool) *LayoutHandler { - return &LayoutHandler{hasTheme: hasTheme, cache: make(map[layoutCacheKey][]string)} +func NewLayoutHandler() *LayoutHandler { + return &LayoutHandler{cache: make(map[layoutCacheKey][]string)} } // For returns a layout for the given LayoutDescriptor and options. @@ -72,30 +70,6 @@ func (l *LayoutHandler) For(d LayoutDescriptor, f Format) ([]string, error) { layouts := resolvePageTemplate(d, f) - if l.hasTheme { - // From Hugo 0.33 we interleave the project/theme templates. This was kind of a fundamental change, but the - // previous behaviour was surprising. - // As an example, an `index.html` in theme for the home page will now win over a `_default/list.html` in the project. - layoutsWithThemeLayouts := []string{} - - // First place all non internal templates - for _, t := range layouts { - if !strings.HasPrefix(t, "_internal/") { - layoutsWithThemeLayouts = append(layoutsWithThemeLayouts, t) - layoutsWithThemeLayouts = append(layoutsWithThemeLayouts, "theme/"+t) - } - } - - // Lastly place internal templates - for _, t := range layouts { - if strings.HasPrefix(t, "_internal/") { - layoutsWithThemeLayouts = append(layoutsWithThemeLayouts, t) - } - } - - layouts = layoutsWithThemeLayouts - } - layouts = prependTextPrefixIfNeeded(f, layouts...) layouts = helpers.UniqueStrings(layouts) diff --git a/output/layout_base.go b/output/layout_base.go index 49ae1d64e..31e1194f4 100644 --- a/output/layout_base.go +++ b/output/layout_base.go @@ -40,26 +40,16 @@ type TemplateNames struct { } type TemplateLookupDescriptor struct { - // TemplateDir is the project or theme root of the current template. - // This will be the same as WorkingDir for non-theme templates. - TemplateDir string - // The full path to the site root. WorkingDir string - // Main project layout dir, defaults to "layouts" - LayoutDir string - // The path to the template relative the the base. // I.e. shortcodes/youtube.html RelPath string - // The template name prefix to look for, i.e. "theme". + // The template name prefix to look for. Prefix string - // The theme dir if theme active. - ThemeDir string - // All the output formats in play. This is used to decide if text/template or // html/template. OutputFormats Formats @@ -71,6 +61,7 @@ type TemplateLookupDescriptor struct { func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { name := filepath.ToSlash(d.RelPath) + name = strings.TrimPrefix(name, "/") if d.Prefix != "" { name = strings.Trim(d.Prefix, "/") + "/" + name @@ -78,22 +69,8 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { var ( id TemplateNames - - // This is the path to the actual template in process. This may - // be in the theme's or the project's /layouts. - baseLayoutDir = filepath.Join(d.TemplateDir, d.LayoutDir) - fullPath = filepath.Join(baseLayoutDir, d.RelPath) - - // This is always the project's layout dir. - baseWorkLayoutDir = filepath.Join(d.WorkingDir, d.LayoutDir) - - baseThemeLayoutDir string ) - if d.ThemeDir != "" { - baseThemeLayoutDir = filepath.Join(d.ThemeDir, "layouts") - } - // The filename will have a suffix with an optional type indicator. // Examples: // index.html @@ -119,7 +96,7 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { filenameNoSuffix := parts[0] - id.OverlayFilename = fullPath + id.OverlayFilename = d.RelPath id.Name = name if isPlainText { @@ -127,7 +104,7 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { } // Ace and Go templates may have both a base and inner template. - pathDir := filepath.Dir(fullPath) + pathDir := filepath.Dir(d.RelPath) if ext == "amber" || strings.HasSuffix(pathDir, "partials") || strings.HasSuffix(pathDir, "shortcodes") { // No base template support @@ -150,7 +127,7 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { // This may be a view that shouldn't have base template // Have to look inside it to make sure - needsBase, err := d.ContainsAny(fullPath, innerMarkers) + needsBase, err := d.ContainsAny(d.RelPath, innerMarkers) if err != nil { return id, err } @@ -158,21 +135,14 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { if needsBase { currBaseFilename := fmt.Sprintf("%s-%s", filenameNoSuffix, baseFilename) - templateDir := filepath.Dir(fullPath) - - // Find the base, e.g. "_default". - baseTemplatedDir := strings.TrimPrefix(templateDir, baseLayoutDir) - baseTemplatedDir = strings.TrimPrefix(baseTemplatedDir, helpers.FilePathSeparator) - // Look for base template in the follwing order: // 1. <current-path>/<template-name>-baseof.<outputFormat>(optional).<suffix>, e.g. list-baseof.<outputFormat>(optional).<suffix>. // 2. <current-path>/baseof.<outputFormat>(optional).<suffix> // 3. _default/<template-name>-baseof.<outputFormat>(optional).<suffix>, e.g. list-baseof.<outputFormat>(optional).<suffix>. // 4. _default/baseof.<outputFormat>(optional).<suffix> - // For each of the steps above, it will first look in the project, then, if theme is set, - // in the theme's layouts folder. - // Also note that the <current-path> may be both the project's layout folder and the theme's. - pairsToCheck := createPairsToCheck(baseTemplatedDir, baseFilename, currBaseFilename) + // + // The filesystem it looks in a a composite of the project and potential theme(s). + pathsToCheck := createPathsToCheck(pathDir, baseFilename, currBaseFilename) // We may have language code and/or "terms" in the template name. We want the most specific, // but need to fall back to the baseof.html or baseof.ace if needed. @@ -183,20 +153,15 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { if len(p1) > 0 && len(p1) == len(p2) { for i := len(p1); i > 0; i-- { v1, v2 := strings.Join(p1[:i], ".")+"."+ext, strings.Join(p2[:i], ".")+"."+ext - pairsToCheck = append(pairsToCheck, createPairsToCheck(baseTemplatedDir, v1, v2)...) + pathsToCheck = append(pathsToCheck, createPathsToCheck(pathDir, v1, v2)...) } } - Loop: - for _, pair := range pairsToCheck { - pathsToCheck := basePathsToCheck(pair, baseLayoutDir, baseWorkLayoutDir, baseThemeLayoutDir) - - for _, pathToCheck := range pathsToCheck { - if ok, err := d.FileExists(pathToCheck); err == nil && ok { - id.MasterFilename = pathToCheck - break Loop - } + for _, p := range pathsToCheck { + if ok, err := d.FileExists(p); err == nil && ok { + id.MasterFilename = p + break } } } @@ -205,29 +170,11 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { } -func createPairsToCheck(baseTemplatedDir, baseFilename, currBaseFilename string) [][]string { - return [][]string{ - {baseTemplatedDir, currBaseFilename}, - {baseTemplatedDir, baseFilename}, - {"_default", currBaseFilename}, - {"_default", baseFilename}, - } -} - -func basePathsToCheck(path []string, layoutDir, workLayoutDir, themeLayoutDir string) []string { - // workLayoutDir will always be the most specific, so start there. - pathsToCheck := []string{filepath.Join((append([]string{workLayoutDir}, path...))...)} - - if layoutDir != "" && layoutDir != workLayoutDir { - pathsToCheck = append(pathsToCheck, filepath.Join((append([]string{layoutDir}, path...))...)) +func createPathsToCheck(baseTemplatedDir, baseFilename, currBaseFilename string) []string { + return []string{ + filepath.Join(baseTemplatedDir, currBaseFilename), + filepath.Join(baseTemplatedDir, baseFilename), + filepath.Join("_default", currBaseFilename), + filepath.Join("_default", baseFilename), } - - // May have a theme - if themeLayoutDir != "" && themeLayoutDir != layoutDir { - pathsToCheck = append(pathsToCheck, filepath.Join((append([]string{themeLayoutDir}, path...))...)) - - } - - return pathsToCheck - } diff --git a/output/layout_base_test.go b/output/layout_base_test.go index d7c7fbb90..719407524 100644 --- a/output/layout_base_test.go +++ b/output/layout_base_test.go @@ -25,8 +25,6 @@ func TestLayoutBase(t *testing.T) { var ( workingDir = "/sites/mysite/" - themeDir = "/themes/mytheme/" - layoutBase1 = "layouts" layoutPath1 = "_default/single.html" layoutPathAmp = "_default/single.amp.html" layoutPathJSON = "_default/single.json" @@ -39,108 +37,72 @@ func TestLayoutBase(t *testing.T) { basePathMatchStrings string expect TemplateNames }{ - {"No base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1}, false, "", + {"No base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPath1}, false, "", TemplateNames{ Name: "_default/single.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.html", + OverlayFilename: "_default/single.html", }}, - {"Base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1}, true, "", + {"Base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPath1}, true, "", TemplateNames{ Name: "_default/single.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.html", - MasterFilename: "/sites/mysite/layouts/_default/single-baseof.html", + OverlayFilename: "_default/single.html", + MasterFilename: "_default/single-baseof.html", }}, // Issue #3893 - {"Base Lang, Default Base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: "layouts", RelPath: "_default/list.en.html"}, true, "_default/baseof.html", + {"Base Lang, Default Base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: "_default/list.en.html"}, true, "_default/baseof.html", TemplateNames{ Name: "_default/list.en.html", - OverlayFilename: "/sites/mysite/layouts/_default/list.en.html", - MasterFilename: "/sites/mysite/layouts/_default/baseof.html", + OverlayFilename: "_default/list.en.html", + MasterFilename: "_default/baseof.html", }}, - {"Base Lang, Lang Base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: "layouts", RelPath: "_default/list.en.html"}, true, "_default/baseof.html|_default/baseof.en.html", + {"Base Lang, Lang Base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: "_default/list.en.html"}, true, "_default/baseof.html|_default/baseof.en.html", TemplateNames{ Name: "_default/list.en.html", - OverlayFilename: "/sites/mysite/layouts/_default/list.en.html", - MasterFilename: "/sites/mysite/layouts/_default/baseof.en.html", + OverlayFilename: "_default/list.en.html", + MasterFilename: "_default/baseof.en.html", }}, // Issue #3856 - {"Base Taxonomy Term", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: "taxonomy/tag.terms.html"}, true, "_default/baseof.html", + {"Base Taxonomy Term", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: "taxonomy/tag.terms.html"}, true, "_default/baseof.html", TemplateNames{ Name: "taxonomy/tag.terms.html", - OverlayFilename: "/sites/mysite/layouts/taxonomy/tag.terms.html", - MasterFilename: "/sites/mysite/layouts/_default/baseof.html", + OverlayFilename: "taxonomy/tag.terms.html", + MasterFilename: "_default/baseof.html", }}, - {"Base in theme", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, ThemeDir: themeDir}, true, - "mytheme/layouts/_default/baseof.html", - TemplateNames{ - Name: "_default/single.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.html", - MasterFilename: "/themes/mytheme/layouts/_default/baseof.html", - }}, - {"Template in theme, base in theme", TemplateLookupDescriptor{TemplateDir: themeDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, ThemeDir: themeDir}, true, - "mytheme/layouts/_default/baseof.html", - TemplateNames{ - Name: "_default/single.html", - OverlayFilename: "/themes/mytheme/layouts/_default/single.html", - MasterFilename: "/themes/mytheme/layouts/_default/baseof.html", - }}, - {"Template in theme, base in site", TemplateLookupDescriptor{TemplateDir: themeDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, ThemeDir: themeDir}, true, - "/sites/mysite/layouts/_default/baseof.html", - TemplateNames{ - Name: "_default/single.html", - OverlayFilename: "/themes/mytheme/layouts/_default/single.html", - MasterFilename: "/sites/mysite/layouts/_default/baseof.html", - }}, - {"Template in site, base in theme", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, ThemeDir: themeDir}, true, - "/themes/mytheme", - TemplateNames{ - Name: "_default/single.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.html", - MasterFilename: "/themes/mytheme/layouts/_default/single-baseof.html", - }}, - {"With prefix, base in theme", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, - ThemeDir: themeDir, Prefix: "someprefix"}, true, - "mytheme/layouts/_default/baseof.html", - TemplateNames{ - Name: "someprefix/_default/single.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.html", - MasterFilename: "/themes/mytheme/layouts/_default/baseof.html", - }}, - {"Partial", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: "partials/menu.html"}, true, + {"Partial", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: "partials/menu.html"}, true, "mytheme/layouts/_default/baseof.html", TemplateNames{ Name: "partials/menu.html", - OverlayFilename: "/sites/mysite/layouts/partials/menu.html", + OverlayFilename: "partials/menu.html", }}, - {"AMP, no base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathAmp}, false, "", + {"AMP, no base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathAmp}, false, "", TemplateNames{ Name: "_default/single.amp.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.amp.html", + OverlayFilename: "_default/single.amp.html", }}, - {"JSON, no base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathJSON}, false, "", + {"JSON, no base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathJSON}, false, "", TemplateNames{ Name: "_default/single.json", - OverlayFilename: "/sites/mysite/layouts/_default/single.json", + OverlayFilename: "_default/single.json", }}, - {"AMP with base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathAmp}, true, "single-baseof.html|single-baseof.amp.html", + {"AMP with base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathAmp}, true, "single-baseof.html|single-baseof.amp.html", TemplateNames{ Name: "_default/single.amp.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.amp.html", - MasterFilename: "/sites/mysite/layouts/_default/single-baseof.amp.html", + OverlayFilename: "_default/single.amp.html", + MasterFilename: "_default/single-baseof.amp.html", }}, - {"AMP with no AMP base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathAmp}, true, "single-baseof.html", + {"AMP with no AMP base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathAmp}, true, "single-baseof.html", TemplateNames{ Name: "_default/single.amp.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.amp.html", - MasterFilename: "/sites/mysite/layouts/_default/single-baseof.html", + OverlayFilename: "_default/single.amp.html", + MasterFilename: "_default/single-baseof.html", }}, - {"JSON with base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathJSON}, true, "single-baseof.json", + {"JSON with base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathJSON}, true, "single-baseof.json", TemplateNames{ Name: "_default/single.json", - OverlayFilename: "/sites/mysite/layouts/_default/single.json", - MasterFilename: "/sites/mysite/layouts/_default/single-baseof.json", + OverlayFilename: "_default/single.json", + MasterFilename: "_default/single-baseof.json", }}, } { t.Run(this.name, func(t *testing.T) { @@ -164,7 +126,6 @@ func TestLayoutBase(t *testing.T) { this.d.OutputFormats = Formats{AMPFormat, HTMLFormat, RSSFormat, JSONFormat} this.d.WorkingDir = filepath.FromSlash(this.d.WorkingDir) - this.d.LayoutDir = filepath.FromSlash(this.d.LayoutDir) this.d.RelPath = filepath.FromSlash(this.d.RelPath) this.d.ContainsAny = needsBase this.d.FileExists = fileExists diff --git a/output/layout_test.go b/output/layout_test.go index 3c7fde41a..4b958e9ff 100644 --- a/output/layout_test.go +++ b/output/layout_test.go @@ -57,62 +57,61 @@ func TestLayout(t *testing.T) { for _, this := range []struct { name string d LayoutDescriptor - hasTheme bool layoutOverride string tp Format expect []string expectCount int }{ - {"Home", LayoutDescriptor{Kind: "home"}, true, "", ampType, - []string{"index.amp.html", "theme/index.amp.html", "home.amp.html", "theme/home.amp.html", "list.amp.html", "theme/list.amp.html", "index.html", "theme/index.html", "home.html", "theme/home.html", "list.html", "theme/list.html", "_default/index.amp.html"}, 24}, - {"Home, HTML", LayoutDescriptor{Kind: "home"}, true, "", htmlFormat, + {"Home", LayoutDescriptor{Kind: "home"}, "", ampType, + []string{"index.amp.html", "home.amp.html", "list.amp.html", "index.html", "home.html", "list.html", "_default/index.amp.html"}, 12}, + {"Home, HTML", LayoutDescriptor{Kind: "home"}, "", htmlFormat, // We will eventually get to index.html. This looks stuttery, but makes the lookup logic easy to understand. - []string{"index.html.html", "theme/index.html.html", "home.html.html"}, 24}, - {"Home, french language", LayoutDescriptor{Kind: "home", Lang: "fr"}, true, "", ampType, - []string{"index.fr.amp.html", "theme/index.fr.amp.html"}, - 48}, - {"Home, no ext or delim", LayoutDescriptor{Kind: "home"}, true, "", noExtDelimFormat, - []string{"index.nem", "theme/index.nem", "home.nem", "theme/home.nem", "list.nem"}, 12}, - {"Home, no ext", LayoutDescriptor{Kind: "home"}, true, "", noExt, - []string{"index.nex", "theme/index.nex", "home.nex", "theme/home.nex", "list.nex"}, 12}, - {"Page, no ext or delim", LayoutDescriptor{Kind: "page"}, true, "", noExtDelimFormat, - []string{"_default/single.nem", "theme/_default/single.nem"}, 2}, - {"Section", LayoutDescriptor{Kind: "section", Section: "sect1"}, false, "", ampType, + []string{"index.html.html", "home.html.html"}, 12}, + {"Home, french language", LayoutDescriptor{Kind: "home", Lang: "fr"}, "", ampType, + []string{"index.fr.amp.html"}, + 24}, + {"Home, no ext or delim", LayoutDescriptor{Kind: "home"}, "", noExtDelimFormat, + []string{"index.nem", "home.nem", "list.nem"}, 6}, + {"Home, no ext", LayoutDescriptor{Kind: "home"}, "", noExt, + []string{"index.nex", "home.nex", "list.nex"}, 6}, + {"Page, no ext or delim", LayoutDescriptor{Kind: "page"}, "", noExtDelimFormat, + []string{"_default/single.nem"}, 1}, + {"Section", LayoutDescriptor{Kind: "section", Section: "sect1"}, "", ampType, []string{"sect1/sect1.amp.html", "sect1/section.amp.html", "sect1/list.amp.html", "sect1/sect1.html", "sect1/section.html", "sect1/list.html", "section/sect1.amp.html", "section/section.amp.html"}, 18}, - {"Section with layout", LayoutDescriptor{Kind: "section", Section: "sect1", Layout: "mylayout"}, false, "", ampType, + {"Section with layout", LayoutDescriptor{Kind: "section", Section: "sect1", Layout: "mylayout"}, "", ampType, []string{"sect1/mylayout.amp.html", "sect1/sect1.amp.html", "sect1/section.amp.html", "sect1/list.amp.html", "sect1/mylayout.html", "sect1/sect1.html"}, 24}, - {"Taxonomy", LayoutDescriptor{Kind: "taxonomy", Section: "tag"}, false, "", ampType, + {"Taxonomy", LayoutDescriptor{Kind: "taxonomy", Section: "tag"}, "", ampType, []string{"taxonomy/tag.amp.html", "taxonomy/taxonomy.amp.html", "taxonomy/list.amp.html", "taxonomy/tag.html", "taxonomy/taxonomy.html"}, 18}, - {"Taxonomy term", LayoutDescriptor{Kind: "taxonomyTerm", Section: "categories"}, false, "", ampType, + {"Taxonomy term", LayoutDescriptor{Kind: "taxonomyTerm", Section: "categories"}, "", ampType, []string{"taxonomy/categories.terms.amp.html", "taxonomy/terms.amp.html", "taxonomy/list.amp.html", "taxonomy/categories.terms.html", "taxonomy/terms.html"}, 18}, - {"Page", LayoutDescriptor{Kind: "page"}, true, "", ampType, - []string{"_default/single.amp.html", "theme/_default/single.amp.html", "_default/single.html", "theme/_default/single.html"}, 4}, - {"Page with layout", LayoutDescriptor{Kind: "page", Layout: "mylayout"}, false, "", ampType, + {"Page", LayoutDescriptor{Kind: "page"}, "", ampType, + []string{"_default/single.amp.html", "_default/single.html"}, 2}, + {"Page with layout", LayoutDescriptor{Kind: "page", Layout: "mylayout"}, "", ampType, []string{"_default/mylayout.amp.html", "_default/single.amp.html", "_default/mylayout.html", "_default/single.html"}, 4}, - {"Page with layout and type", LayoutDescriptor{Kind: "page", Layout: "mylayout", Type: "myttype"}, false, "", ampType, + {"Page with layout and type", LayoutDescriptor{Kind: "page", Layout: "mylayout", Type: "myttype"}, "", ampType, []string{"myttype/mylayout.amp.html", "myttype/single.amp.html", "myttype/mylayout.html"}, 8}, - {"Page with layout and type with subtype", LayoutDescriptor{Kind: "page", Layout: "mylayout", Type: "myttype/mysubtype"}, false, "", ampType, + {"Page with layout and type with subtype", LayoutDescriptor{Kind: "page", Layout: "mylayout", Type: "myttype/mysubtype"}, "", ampType, []string{"myttype/mysubtype/mylayout.amp.html", "myttype/mysubtype/single.amp.html", "myttype/mysubtype/mylayout.html"}, 8}, // RSS - {"RSS Home with theme", LayoutDescriptor{Kind: "home"}, true, "", RSSFormat, - []string{"index.rss.xml", "theme/index.rss.xml", "home.rss.xml", "theme/home.rss.xml", "rss.xml"}, 29}, - {"RSS Section", LayoutDescriptor{Kind: "section", Section: "sect1"}, false, "", RSSFormat, + {"RSS Home", LayoutDescriptor{Kind: "home"}, "", RSSFormat, + []string{"index.rss.xml", "home.rss.xml", "rss.xml"}, 15}, + {"RSS Section", LayoutDescriptor{Kind: "section", Section: "sect1"}, "", RSSFormat, []string{"sect1/sect1.rss.xml", "sect1/section.rss.xml", "sect1/rss.xml", "sect1/list.rss.xml", "sect1/sect1.xml", "sect1/section.xml"}, 22}, - {"RSS Taxonomy", LayoutDescriptor{Kind: "taxonomy", Section: "tag"}, false, "", RSSFormat, + {"RSS Taxonomy", LayoutDescriptor{Kind: "taxonomy", Section: "tag"}, "", RSSFormat, []string{"taxonomy/tag.rss.xml", "taxonomy/taxonomy.rss.xml", "taxonomy/rss.xml", "taxonomy/list.rss.xml", "taxonomy/tag.xml", "taxonomy/taxonomy.xml"}, 22}, - {"RSS Taxonomy term", LayoutDescriptor{Kind: "taxonomyTerm", Section: "tag"}, false, "", RSSFormat, + {"RSS Taxonomy term", LayoutDescriptor{Kind: "taxonomyTerm", Section: "tag"}, "", RSSFormat, []string{"taxonomy/tag.terms.rss.xml", "taxonomy/terms.rss.xml", "taxonomy/rss.xml", "taxonomy/list.rss.xml", "taxonomy/tag.terms.xml"}, 22}, - {"Home plain text", LayoutDescriptor{Kind: "home"}, true, "", JSONFormat, - []string{"_text/index.json.json", "_text/theme/index.json.json", "_text/home.json.json", "_text/theme/home.json.json"}, 24}, - {"Page plain text", LayoutDescriptor{Kind: "page"}, true, "", JSONFormat, - []string{"_text/_default/single.json.json", "_text/theme/_default/single.json.json", "_text/_default/single.json", "_text/theme/_default/single.json"}, 4}, - {"Reserved section, shortcodes", LayoutDescriptor{Kind: "section", Section: "shortcodes", Type: "shortcodes"}, true, "", ampType, - []string{"section/shortcodes.amp.html", "theme/section/shortcodes.amp.html"}, 24}, - {"Reserved section, partials", LayoutDescriptor{Kind: "section", Section: "partials", Type: "partials"}, true, "", ampType, - []string{"section/partials.amp.html", "theme/section/partials.amp.html"}, 24}, + {"Home plain text", LayoutDescriptor{Kind: "home"}, "", JSONFormat, + []string{"_text/index.json.json", "_text/home.json.json"}, 12}, + {"Page plain text", LayoutDescriptor{Kind: "page"}, "", JSONFormat, + []string{"_text/_default/single.json.json", "_text/_default/single.json"}, 2}, + {"Reserved section, shortcodes", LayoutDescriptor{Kind: "section", Section: "shortcodes", Type: "shortcodes"}, "", ampType, + []string{"section/shortcodes.amp.html"}, 12}, + {"Reserved section, partials", LayoutDescriptor{Kind: "section", Section: "partials", Type: "partials"}, "", ampType, + []string{"section/partials.amp.html"}, 12}, } { t.Run(this.name, func(t *testing.T) { - l := NewLayoutHandler(this.hasTheme) + l := NewLayoutHandler() layouts, err := l.For(this.d, this.tp) @@ -130,11 +129,6 @@ func TestLayout(t *testing.T) { } - if !this.hasTheme { - for _, layout := range layouts { - require.NotContains(t, layout, "theme") - } - } }) } @@ -142,7 +136,7 @@ func TestLayout(t *testing.T) { func BenchmarkLayout(b *testing.B) { descriptor := LayoutDescriptor{Kind: "taxonomyTerm", Section: "categories"} - l := NewLayoutHandler(false) + l := NewLayoutHandler() for i := 0; i < b.N; i++ { layouts, err := l.For(descriptor, HTMLFormat) diff --git a/resource/resource.go b/resource/resource.go index 0714805e8..9a3725f8a 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -23,6 +23,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/common/maps" + "github.com/spf13/afero" "github.com/spf13/cast" @@ -282,7 +284,6 @@ func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) { if err != nil { return nil, err } - s.GetLayoutDirPath() genImagePath := filepath.FromSlash("_gen/images") @@ -644,7 +645,7 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) er if found { m := cast.ToStringMap(params) // Needed for case insensitive fetching of params values - helpers.ToLowerMap(m) + maps.ToLower(m) ma.updateParams(m) } } diff --git a/resource/testhelpers_test.go b/resource/testhelpers_test.go index 9b50633bd..360adc038 100644 --- a/resource/testhelpers_test.go +++ b/resource/testhelpers_test.go @@ -30,6 +30,10 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) * cfg.Set("baseURL", baseURL) cfg.Set("resourceDir", "resources") cfg.Set("contentDir", "content") + cfg.Set("dataDir", "data") + cfg.Set("i18nDir", "i18n") + cfg.Set("layoutDir", "layouts") + cfg.Set("archetypeDir", "archetypes") imagingCfg := map[string]interface{}{ "resampleFilter": "linear", @@ -63,8 +67,12 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { } cfg.Set("workingDir", workDir) - cfg.Set("contentDir", filepath.Join(workDir, "content")) cfg.Set("resourceDir", filepath.Join(workDir, "res")) + cfg.Set("contentDir", "content") + cfg.Set("dataDir", "data") + cfg.Set("i18nDir", "i18n") + cfg.Set("layoutDir", "layouts") + cfg.Set("archetypeDir", "archetypes") fs := hugofs.NewFrom(hugofs.Os, cfg) fs.Destination = &afero.MemMapFs{} diff --git a/source/content_directory_test.go b/source/content_directory_test.go index ed00af625..7f050e0da 100644 --- a/source/content_directory_test.go +++ b/source/content_directory_test.go @@ -20,7 +20,6 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -52,9 +51,7 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) { } for i, test := range tests { - - v := viper.New() - v.Set("contentDir", "content") + v := newTestConfig() v.Set("ignoreFiles", test.ignoreFilesRegexpes) fs := hugofs.NewMem(v) ps, err := helpers.NewPathSpec(fs, v) diff --git a/source/dirs.go b/source/dirs.go deleted file mode 100644 index 49a849453..000000000 --- a/source/dirs.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "errors" - "os" - "path/filepath" - "strings" - - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" - jww "github.com/spf13/jwalterweatherman" -) - -// Dirs holds the source directories for a given build. -// In case where there are more than one of a kind, the order matters: -// It will be used to construct a union filesystem, so the right-most directory -// will "win" on duplicates. Typically, the theme version will be the first. -type Dirs struct { - logger *jww.Notepad - pathSpec *helpers.PathSpec - - staticDirs []string - AbsStaticDirs []string - - Language *helpers.Language -} - -// NewDirs creates a new dirs with the given configuration and filesystem. -func NewDirs(fs *hugofs.Fs, cfg config.Provider, logger *jww.Notepad) (*Dirs, error) { - ps, err := helpers.NewPathSpec(fs, cfg) - if err != nil { - return nil, err - } - - var l *helpers.Language - if language, ok := cfg.(*helpers.Language); ok { - l = language - } - - d := &Dirs{Language: l, pathSpec: ps, logger: logger} - - return d, d.init(cfg) - -} - -func (d *Dirs) init(cfg config.Provider) error { - - var ( - statics []string - ) - - if d.pathSpec.Theme() != "" { - statics = append(statics, filepath.Join(d.pathSpec.ThemesDir(), d.pathSpec.Theme(), "static")) - } - - _, isLanguage := cfg.(*helpers.Language) - languages, hasLanguages := cfg.Get("languagesSorted").(helpers.Languages) - - if !isLanguage && !hasLanguages { - return errors.New("missing languagesSorted in config") - } - - if !isLanguage { - // Merge all the static dirs. - for _, l := range languages { - addend, err := d.staticDirsFor(l) - if err != nil { - return err - } - - statics = append(statics, addend...) - } - } else { - addend, err := d.staticDirsFor(cfg) - if err != nil { - return err - } - - statics = append(statics, addend...) - } - - d.staticDirs = removeDuplicatesKeepRight(statics) - d.AbsStaticDirs = make([]string, len(d.staticDirs)) - for i, di := range d.staticDirs { - d.AbsStaticDirs[i] = d.pathSpec.AbsPathify(di) + helpers.FilePathSeparator - } - - return nil -} - -func (d *Dirs) staticDirsFor(cfg config.Provider) ([]string, error) { - var statics []string - ps, err := helpers.NewPathSpec(d.pathSpec.Fs, cfg) - if err != nil { - return statics, err - } - - statics = append(statics, ps.StaticDirs()...) - - return statics, nil -} - -// CreateStaticFs will create a union filesystem with the static paths configured. -// Any missing directories will be logged as warnings. -func (d *Dirs) CreateStaticFs() (afero.Fs, error) { - var ( - source = d.pathSpec.Fs.Source - absPaths []string - ) - - for _, staticDir := range d.AbsStaticDirs { - if _, err := source.Stat(staticDir); os.IsNotExist(err) { - d.logger.WARN.Printf("Unable to find Static Directory: %s", staticDir) - } else { - absPaths = append(absPaths, staticDir) - } - - } - - if len(absPaths) == 0 { - return nil, nil - } - - return d.createOverlayFs(absPaths), nil - -} - -// IsStatic returns whether the given filename is located in one of the static -// source dirs. -func (d *Dirs) IsStatic(filename string) bool { - for _, absPath := range d.AbsStaticDirs { - if strings.HasPrefix(filename, absPath) { - return true - } - } - return false -} - -// MakeStaticPathRelative creates a relative path from the given filename. -// It will return an empty string if the filename is not a member of dirs. -func (d *Dirs) MakeStaticPathRelative(filename string) string { - for _, currentPath := range d.AbsStaticDirs { - if strings.HasPrefix(filename, currentPath) { - return strings.TrimPrefix(filename, currentPath) - } - } - - return "" - -} - -func (d *Dirs) createOverlayFs(absPaths []string) afero.Fs { - source := d.pathSpec.Fs.Source - - if len(absPaths) == 1 { - return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])) - } - - base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])) - overlay := d.createOverlayFs(absPaths[1:]) - - return afero.NewCopyOnWriteFs(base, overlay) -} - -func removeDuplicatesKeepRight(in []string) []string { - seen := make(map[string]bool) - var out []string - for i := len(in) - 1; i >= 0; i-- { - v := in[i] - if seen[v] { - continue - } - out = append([]string{v}, out...) - seen[v] = true - } - - return out -} diff --git a/source/dirs_test.go b/source/dirs_test.go deleted file mode 100644 index 46236120e..000000000 --- a/source/dirs_test.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "testing" - - "github.com/gohugoio/hugo/helpers" - - "fmt" - - "io/ioutil" - "log" - "os" - "path/filepath" - - "github.com/gohugoio/hugo/config" - "github.com/spf13/afero" - - jww "github.com/spf13/jwalterweatherman" - - "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" -) - -var logger = jww.NewNotepad(jww.LevelInfo, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) - -func TestStaticDirs(t *testing.T) { - assert := require.New(t) - - tests := []struct { - setup func(cfg config.Provider, fs *hugofs.Fs) config.Provider - expected []string - }{ - - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("staticDir", "s1") - return cfg - }, []string{"s1"}}, - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("staticDir", []string{"s2", "s1", "s2"}) - return cfg - }, []string{"s1", "s2"}}, - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("theme", "mytheme") - cfg.Set("themesDir", "themes") - cfg.Set("staticDir", []string{"s1", "s2"}) - return cfg - }, []string{filepath.FromSlash("themes/mytheme/static"), "s1", "s2"}}, - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("staticDir", "s1") - - l1 := helpers.NewLanguage("en", cfg) - l1.Set("staticDir", []string{"l1s1", "l1s2"}) - return l1 - - }, []string{"l1s1", "l1s2"}}, - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("staticDir", "s1") - - l1 := helpers.NewLanguage("en", cfg) - l1.Set("staticDir2", []string{"l1s1", "l1s2"}) - return l1 - - }, []string{"s1", "l1s1", "l1s2"}}, - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("staticDir", []string{"s1", "s2"}) - - l1 := helpers.NewLanguage("en", cfg) - l1.Set("staticDir2", []string{"l1s1", "l1s2"}) - return l1 - - }, []string{"s1", "s2", "l1s1", "l1s2"}}, - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("staticDir", "s1") - - l1 := helpers.NewLanguage("en", cfg) - l1.Set("staticDir2", []string{"l1s1", "l1s2"}) - l2 := helpers.NewLanguage("nn", cfg) - l2.Set("staticDir3", []string{"l2s1", "l2s2"}) - l2.Set("staticDir", []string{"l2"}) - - cfg.Set("languagesSorted", helpers.Languages{l1, l2}) - return cfg - - }, []string{"s1", "l1s1", "l1s2", "l2", "l2s1", "l2s2"}}, - } - - for i, test := range tests { - msg := fmt.Sprintf("Test %d", i) - v := viper.New() - v.Set("contentDir", "content") - - fs := hugofs.NewMem(v) - cfg := test.setup(v, fs) - cfg.Set("workingDir", filepath.FromSlash("/work")) - _, isLanguage := cfg.(*helpers.Language) - if !isLanguage && !cfg.IsSet("languagesSorted") { - cfg.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(cfg)}) - } - dirs, err := NewDirs(fs, cfg, logger) - assert.NoError(err) - assert.Equal(test.expected, dirs.staticDirs, msg) - assert.Len(dirs.AbsStaticDirs, len(dirs.staticDirs)) - - for i, d := range dirs.staticDirs { - abs := dirs.AbsStaticDirs[i] - assert.Equal(filepath.Join("/work", d)+helpers.FilePathSeparator, abs) - assert.True(dirs.IsStatic(filepath.Join(abs, "logo.png"))) - rel := dirs.MakeStaticPathRelative(filepath.Join(abs, "logo.png")) - assert.Equal("logo.png", rel) - } - - assert.False(dirs.IsStatic(filepath.FromSlash("/some/other/dir/logo.png"))) - - } - -} - -func TestStaticDirsFs(t *testing.T) { - assert := require.New(t) - v := viper.New() - fs := hugofs.NewMem(v) - v.Set("workingDir", filepath.FromSlash("/work")) - v.Set("theme", "mytheme") - v.Set("themesDir", "themes") - v.Set("contentDir", "content") - v.Set("staticDir", []string{"s1", "s2"}) - v.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(v)}) - - writeToFs(t, fs.Source, "/work/s1/f1.txt", "s1-f1") - writeToFs(t, fs.Source, "/work/s2/f2.txt", "s2-f2") - writeToFs(t, fs.Source, "/work/s1/f2.txt", "s1-f2") - writeToFs(t, fs.Source, "/work/themes/mytheme/static/f1.txt", "theme-f1") - writeToFs(t, fs.Source, "/work/themes/mytheme/static/f3.txt", "theme-f3") - - dirs, err := NewDirs(fs, v, logger) - assert.NoError(err) - - sfs, err := dirs.CreateStaticFs() - assert.NoError(err) - - assert.Equal("s1-f1", readFileFromFs(t, sfs, "f1.txt")) - assert.Equal("s2-f2", readFileFromFs(t, sfs, "f2.txt")) - assert.Equal("theme-f3", readFileFromFs(t, sfs, "f3.txt")) - -} - -func TestRemoveDuplicatesKeepRight(t *testing.T) { - in := []string{"a", "b", "c", "a"} - out := removeDuplicatesKeepRight(in) - - require.Equal(t, []string{"b", "c", "a"}, out) -} - -func writeToFs(t testing.TB, fs afero.Fs, filename, content string) { - if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil { - t.Fatalf("Failed to write file: %s", err) - } -} - -func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { - filename = filepath.FromSlash(filename) - b, err := afero.ReadFile(fs, filename) - if err != nil { - afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error { - fmt.Println(" ", path, " ", info) - return nil - }) - t.Fatalf("Failed to read file: %s", err) - } - return string(b) -} diff --git a/source/fileInfo.go b/source/fileInfo.go index 9adb96df4..31885bfd4 100644 --- a/source/fileInfo.go +++ b/source/fileInfo.go @@ -220,7 +220,7 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f // Open implements ReadableFile. func (fi *FileInfo) Open() (io.ReadCloser, error) { - f, err := fi.sp.PathSpec.Fs.Source.Open(fi.Filename()) + f, err := fi.sp.SourceFs.Open(fi.Filename()) return f, err } diff --git a/source/fileInfo_test.go b/source/fileInfo_test.go index ec2a17c65..9d3566240 100644 --- a/source/fileInfo_test.go +++ b/source/fileInfo_test.go @@ -19,8 +19,6 @@ import ( "github.com/gohugoio/hugo/helpers" - "github.com/spf13/viper" - "github.com/gohugoio/hugo/hugofs" "github.com/spf13/afero" "github.com/stretchr/testify/require" @@ -72,14 +70,13 @@ func TestFileInfoLanguage(t *testing.T) { m := afero.NewMemMapFs() lfs := hugofs.NewLanguageFs("sv", langs, m) - v := viper.New() - v.Set("contentDir", "content") + v := newTestConfig() fs := hugofs.NewFrom(m, v) ps, err := helpers.NewPathSpec(fs, v) assert.NoError(err) - s := SourceSpec{Fs: lfs, PathSpec: ps} + s := SourceSpec{SourceFs: lfs, PathSpec: ps} s.Languages = map[string]interface{}{ "en": true, } diff --git a/source/filesystem.go b/source/filesystem.go index 50075e3c4..3f4bf0ff1 100644 --- a/source/filesystem.go +++ b/source/filesystem.go @@ -79,16 +79,13 @@ func (f *Filesystem) captureFiles() { return err } - if f.Fs == nil { + if f.SourceFs == nil { panic("Must have a fs") } - err := helpers.SymbolicWalk(f.Fs, f.Base, walker) + err := helpers.SymbolicWalk(f.SourceFs, f.Base, walker) if err != nil { jww.ERROR.Println(err) - if err == helpers.ErrPathTooShort { - panic("The root path is too short. If this is a test, make sure to init the content paths.") - } } } @@ -100,7 +97,7 @@ func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) { jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err) return false, nil } - linkfi, err := f.Fs.Stat(link) + linkfi, err := f.SourceFs.Stat(link) if err != nil { jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err) return false, nil diff --git a/source/filesystem_test.go b/source/filesystem_test.go index 82f02d404..ee86c1487 100644 --- a/source/filesystem_test.go +++ b/source/filesystem_test.go @@ -19,7 +19,6 @@ import ( "testing" "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" "github.com/spf13/viper" @@ -69,9 +68,19 @@ func TestUnicodeNorm(t *testing.T) { } -func newTestSourceSpec() SourceSpec { +func newTestConfig() *viper.Viper { v := viper.New() v.Set("contentDir", "content") - ps, _ := helpers.NewPathSpec(hugofs.NewMem(v), v) - return SourceSpec{Fs: hugofs.NewMem(v).Source, PathSpec: ps} + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") + return v +} + +func newTestSourceSpec() *SourceSpec { + v := newTestConfig() + fs := hugofs.NewMem(v) + ps, _ := helpers.NewPathSpec(fs, v) + return NewSourceSpec(ps, fs.Source) } diff --git a/source/sourceSpec.go b/source/sourceSpec.go index 634306e5f..144d86ca3 100644 --- a/source/sourceSpec.go +++ b/source/sourceSpec.go @@ -18,6 +18,7 @@ import ( "path/filepath" "regexp" + "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" "github.com/gohugoio/hugo/helpers" @@ -29,7 +30,7 @@ import ( type SourceSpec struct { *helpers.PathSpec - Fs afero.Fs + SourceFs afero.Fs // This is set if the ignoreFiles config is set. ignoreFilesRe []*regexp.Regexp @@ -52,7 +53,7 @@ func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec { } if len(languages) == 0 { - l := helpers.NewDefaultLanguage(cfg) + l := langs.NewDefaultLanguage(cfg) languages[l.Lang] = l defaultLang = l.Lang } @@ -71,12 +72,13 @@ func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec { } } - return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet} + return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, SourceFs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet} + } func (s *SourceSpec) IgnoreFile(filename string) bool { if filename == "" { - if _, ok := s.Fs.(*afero.OsFs); ok { + if _, ok := s.SourceFs.(*afero.OsFs); ok { return true } return false @@ -108,7 +110,7 @@ func (s *SourceSpec) IgnoreFile(filename string) bool { } func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) { - fi, err := helpers.LstatIfPossible(s.Fs, filename) + fi, err := helpers.LstatIfPossible(s.SourceFs, filename) if err != nil { return false, err } @@ -119,7 +121,7 @@ func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) { if fi.Mode()&os.ModeSymlink == os.ModeSymlink { link, err := filepath.EvalSymlinks(filename) - fi, err = helpers.LstatIfPossible(s.Fs, link) + fi, err = helpers.LstatIfPossible(s.SourceFs, link) if err != nil { return false, err } diff --git a/tpl/collections/collections_test.go b/tpl/collections/collections_test.go index 68e7c59d6..ac2c2fe63 100644 --- a/tpl/collections/collections_test.go +++ b/tpl/collections/collections_test.go @@ -29,6 +29,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/langs" jww "github.com/spf13/jwalterweatherman" "github.com/spf13/viper" "github.com/stretchr/testify/assert" @@ -774,7 +775,7 @@ type TstX struct { } func newDeps(cfg config.Provider) *deps.Deps { - l := helpers.NewLanguage("en", cfg) + l := langs.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") cs, err := helpers.NewContentSpec(l) if err != nil { diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go index 79e9b3907..f6baae18b 100644 --- a/tpl/data/resources_test.go +++ b/tpl/data/resources_test.go @@ -27,6 +27,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" "github.com/spf13/viper" "github.com/stretchr/testify/assert" @@ -164,7 +165,7 @@ func TestScpGetRemoteParallel(t *testing.T) { } func newDeps(cfg config.Provider) *deps.Deps { - l := helpers.NewLanguage("en", cfg) + l := langs.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") cs, err := helpers.NewContentSpec(l) if err != nil { diff --git a/tpl/template.go b/tpl/template.go index bdb917ba9..e04d2cc6c 100644 --- a/tpl/template.go +++ b/tpl/template.go @@ -35,7 +35,7 @@ type TemplateHandler interface { TemplateFinder AddTemplate(name, tpl string) error AddLateTemplate(name, tpl string) error - LoadTemplates(absPath, prefix string) + LoadTemplates(prefix string) PrintErrors() MarkReady() diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index 8f91113a8..e838ebc57 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -86,6 +86,10 @@ type templateHandler struct { errors []*templateErr + // This is the filesystem to load the templates from. All the templates are + // stored in the root of this filesystem. + layoutsFs afero.Fs + *deps.Deps } @@ -129,10 +133,11 @@ func (t *templateHandler) Lookup(name string) *tpl.TemplateAdapter { func (t *templateHandler) clone(d *deps.Deps) *templateHandler { c := &templateHandler{ - Deps: d, - html: &htmlTemplates{t: template.Must(t.html.t.Clone()), overlays: make(map[string]*template.Template)}, - text: &textTemplates{t: texttemplate.Must(t.text.t.Clone()), overlays: make(map[string]*texttemplate.Template)}, - errors: make([]*templateErr, 0), + Deps: d, + layoutsFs: d.BaseFs.Layouts.Fs, + html: &htmlTemplates{t: template.Must(t.html.t.Clone()), overlays: make(map[string]*template.Template)}, + text: &textTemplates{t: texttemplate.Must(t.text.t.Clone()), overlays: make(map[string]*texttemplate.Template)}, + errors: make([]*templateErr, 0), } d.Tmpl = c @@ -170,10 +175,11 @@ func newTemplateAdapter(deps *deps.Deps) *templateHandler { overlays: make(map[string]*texttemplate.Template), } return &templateHandler{ - Deps: deps, - html: htmlT, - text: textT, - errors: make([]*templateErr, 0), + Deps: deps, + layoutsFs: deps.BaseFs.Layouts.Fs, + html: htmlT, + text: textT, + errors: make([]*templateErr, 0), } } @@ -208,15 +214,18 @@ func (t *htmlTemplates) Lookup(name string) *tpl.TemplateAdapter { } func (t *htmlTemplates) lookup(name string) *template.Template { - if templ := t.t.Lookup(name); templ != nil { - return templ - } + + // Need to check in the overlay registry first as it will also be found below. if t.overlays != nil { if templ, ok := t.overlays[name]; ok { return templ } } + if templ := t.t.Lookup(name); templ != nil { + return templ + } + if t.clone != nil { return t.clone.Lookup(name) } @@ -248,15 +257,18 @@ func (t *textTemplates) Lookup(name string) *tpl.TemplateAdapter { } func (t *textTemplates) lookup(name string) *texttemplate.Template { - if templ := t.t.Lookup(name); templ != nil { - return templ - } + + // Need to check in the overlay registry first as it will also be found below. if t.overlays != nil { if templ, ok := t.overlays[name]; ok { return templ } } + if templ := t.t.Lookup(name); templ != nil { + return templ + } + if t.clone != nil { return t.clone.Lookup(name) } @@ -287,11 +299,11 @@ func (t *textTemplates) setFuncs(funcMap map[string]interface{}) { t.t.Funcs(funcMap) } -// LoadTemplates loads the templates, starting from the given absolute path. +// LoadTemplates loads the templates from the layouts filesystem. // A prefix can be given to indicate a template namespace to load the templates // into, i.e. "_internal" etc. -func (t *templateHandler) LoadTemplates(absPath, prefix string) { - t.loadTemplates(absPath, prefix) +func (t *templateHandler) LoadTemplates(prefix string) { + t.loadTemplates(prefix) } @@ -406,85 +418,49 @@ func (t *templateHandler) RebuildClone() { t.text.clone = texttemplate.Must(t.text.cloneClone.Clone()) } -func (t *templateHandler) loadTemplates(absPath string, prefix string) { - t.Log.DEBUG.Printf("Load templates from path %q prefix %q", absPath, prefix) +func (t *templateHandler) loadTemplates(prefix string) { walker := func(path string, fi os.FileInfo, err error) error { - if err != nil { + if err != nil || fi.IsDir() { return nil } - t.Log.DEBUG.Println("Template path", path) - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(absPath) - if err != nil { - t.Log.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", absPath, err) - return nil - } - - linkfi, err := t.Fs.Source.Stat(link) - if err != nil { - t.Log.ERROR.Printf("Cannot stat '%s', error was: %s", link, err) - return nil - } - - if !linkfi.Mode().IsRegular() { - t.Log.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", absPath) - } + if isDotFile(path) || isBackupFile(path) || isBaseTemplate(path) { return nil } - if !fi.IsDir() { - if isDotFile(path) || isBackupFile(path) || isBaseTemplate(path) { - return nil - } - - var ( - workingDir = t.PathSpec.WorkingDir() - themeDir = t.PathSpec.GetThemeDir() - layoutDir = t.PathSpec.LayoutDir() - ) - - if themeDir != "" && strings.HasPrefix(absPath, themeDir) { - layoutDir = "layouts" - } - - li := strings.LastIndex(path, layoutDir) + len(layoutDir) + 1 - relPath := path[li:] - templateDir := path[:li-len(layoutDir)-1] - - descriptor := output.TemplateLookupDescriptor{ - TemplateDir: templateDir, - WorkingDir: workingDir, - LayoutDir: layoutDir, - RelPath: relPath, - Prefix: prefix, - ThemeDir: themeDir, - OutputFormats: t.OutputFormatsConfig, - FileExists: func(filename string) (bool, error) { - return helpers.Exists(filename, t.Fs.Source) - }, - ContainsAny: func(filename string, subslices [][]byte) (bool, error) { - return helpers.FileContainsAny(filename, subslices, t.Fs.Source) - }, - } - - tplID, err := output.CreateTemplateNames(descriptor) - if err != nil { - t.Log.ERROR.Printf("Failed to resolve template in path %q: %s", path, err) + workingDir := t.PathSpec.WorkingDir + + descriptor := output.TemplateLookupDescriptor{ + WorkingDir: workingDir, + RelPath: path, + Prefix: prefix, + OutputFormats: t.OutputFormatsConfig, + FileExists: func(filename string) (bool, error) { + return helpers.Exists(filename, t.Layouts.Fs) + }, + ContainsAny: func(filename string, subslices [][]byte) (bool, error) { + return helpers.FileContainsAny(filename, subslices, t.Layouts.Fs) + }, + } - return nil - } + tplID, err := output.CreateTemplateNames(descriptor) + if err != nil { + t.Log.ERROR.Printf("Failed to resolve template in path %q: %s", path, err) - if err := t.addTemplateFile(tplID.Name, tplID.MasterFilename, tplID.OverlayFilename); err != nil { - t.Log.ERROR.Printf("Failed to add template %q in path %q: %s", tplID.Name, path, err) - } + return nil + } + if err := t.addTemplateFile(tplID.Name, tplID.MasterFilename, tplID.OverlayFilename); err != nil { + t.Log.ERROR.Printf("Failed to add template %q in path %q: %s", tplID.Name, path, err) } + return nil } - if err := helpers.SymbolicWalk(t.Fs.Source, absPath, walker); err != nil { + + if err := helpers.SymbolicWalk(t.Layouts.Fs, "", walker); err != nil { t.Log.ERROR.Printf("Failed to load templates: %s", err) } + } func (t *templateHandler) initFuncs() { @@ -534,6 +510,7 @@ func (t *templateHandler) handleMaster(name, overlayFilename, masterFilename str } func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (string, error)) error { + masterTpl := t.lookup(masterFilename) if masterTpl == nil { @@ -565,6 +542,7 @@ func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename strin if err := applyTemplateTransformersToHMLTTemplate(overlayTpl); err != nil { return err } + t.overlays[name] = overlayTpl return err @@ -572,6 +550,7 @@ func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename strin } func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (string, error)) error { + name = strings.TrimPrefix(name, textTmplNamePrefix) masterTpl := t.lookup(masterFilename) @@ -610,12 +589,16 @@ func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename strin func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) error { t.checkState() + t.Log.DEBUG.Printf("Add template file: name %q, baseTemplatePath %q, path %q", name, baseTemplatePath, path) + getTemplate := func(filename string) (string, error) { - b, err := afero.ReadFile(t.Fs.Source, filename) + b, err := afero.ReadFile(t.Layouts.Fs, filename) if err != nil { return "", err } - return string(b), nil + s := string(b) + + return s, nil } // get the suffix and switch on that @@ -625,7 +608,7 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e // Only HTML support for Amber withoutExt := strings.TrimSuffix(name, filepath.Ext(name)) templateName := withoutExt + ".html" - b, err := afero.ReadFile(t.Fs.Source, path) + b, err := afero.ReadFile(t.Layouts.Fs, path) if err != nil { return err @@ -654,14 +637,14 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e case ".ace": // Only HTML support for Ace var innerContent, baseContent []byte - innerContent, err := afero.ReadFile(t.Fs.Source, path) + innerContent, err := afero.ReadFile(t.Layouts.Fs, path) if err != nil { return err } if baseTemplatePath != "" { - baseContent, err = afero.ReadFile(t.Fs.Source, baseTemplatePath) + baseContent, err = afero.ReadFile(t.Layouts.Fs, baseTemplatePath) if err != nil { return err } @@ -680,8 +663,6 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e return err } - t.Log.DEBUG.Printf("Add template file from path %s", path) - return t.AddTemplate(name, templ) } } diff --git a/tpl/tplimpl/template_funcs_test.go b/tpl/tplimpl/template_funcs_test.go index 72842d308..a1745282d 100644 --- a/tpl/tplimpl/template_funcs_test.go +++ b/tpl/tplimpl/template_funcs_test.go @@ -30,6 +30,7 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/i18n" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/tpl/internal" "github.com/gohugoio/hugo/tpl/partials" @@ -43,9 +44,18 @@ var ( logger = jww.NewNotepad(jww.LevelFatal, jww.LevelFatal, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) ) +func newTestConfig() config.Provider { + v := viper.New() + v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") + return v +} + func newDepsConfig(cfg config.Provider) deps.DepsCfg { - l := helpers.NewLanguage("en", cfg) - l.Set("i18nDir", "i18n") + l := langs.NewLanguage("en", cfg) return deps.DepsCfg{ Language: l, Cfg: cfg, @@ -61,13 +71,13 @@ func TestTemplateFuncsExamples(t *testing.T) { workingDir := "/home/hugo" - v := viper.New() + v := newTestConfig() v.Set("workingDir", workingDir) v.Set("multilingual", true) v.Set("contentDir", "content") v.Set("baseURL", "http://mysite.com/hugo/") - v.Set("CurrentContentLanguage", helpers.NewLanguage("en", v)) + v.Set("CurrentContentLanguage", langs.NewLanguage("en", v)) fs := hugofs.NewMem(v) @@ -126,8 +136,7 @@ func TestPartialCached(t *testing.T) { var data struct { } - v := viper.New() - v.Set("contentDir", "content") + v := newTestConfig() config := newDepsConfig(v) diff --git a/tpl/tplimpl/template_test.go b/tpl/tplimpl/template_test.go index 78682e9ab..3ce2a88a2 100644 --- a/tpl/tplimpl/template_test.go +++ b/tpl/tplimpl/template_test.go @@ -18,7 +18,6 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -34,8 +33,7 @@ func TestHTMLEscape(t *testing.T) { "html": "<h1>Hi!</h1>", "other": "<h1>Hi!</h1>", } - v := viper.New() - v.Set("contentDir", "content") + v := newTestConfig() fs := hugofs.NewMem(v) //afero.WriteFile(fs.Source, filepath.Join(workingDir, "README.txt"), []byte("Hugo Rocks!"), 0755) diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index ab3beb804..34de4a6fd 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -22,6 +22,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/langs" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -240,7 +241,7 @@ func TestPlainify(t *testing.T) { } func newDeps(cfg config.Provider) *deps.Deps { - l := helpers.NewLanguage("en", cfg) + l := langs.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") cs, err := helpers.NewContentSpec(l) if err != nil { |