From bb2aa08709c812a5be29922a1a7f4d814e200cab Mon Sep 17 00:00:00 2001 From: Bjørn Erik Pedersen Date: Wed, 16 Jun 2021 19:11:01 +0200 Subject: Implement configuration in a directory for modules Fixes #8654 --- config/configLoader.go | 101 +++++++++++++++++++++++++++++++++++++++++++ hugolib/config.go | 115 +++++-------------------------------------------- hugolib/config_test.go | 53 +++++++++++++++++++++++ modules/client.go | 3 ++ modules/collect.go | 35 +++++++++++---- modules/module.go | 14 +++--- 6 files changed, 201 insertions(+), 120 deletions(-) diff --git a/config/configLoader.go b/config/configLoader.go index 0998b1bef..8dcfcbdcc 100644 --- a/config/configLoader.go +++ b/config/configLoader.go @@ -14,9 +14,14 @@ package config import ( + "os" "path/filepath" "strings" + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/parser/metadecoders" "github.com/spf13/afero" @@ -84,6 +89,102 @@ func loadConfigFromFile(fs afero.Fs, filename string) (map[string]interface{}, e return m, nil } +func LoadConfigFromDir(sourceFs afero.Fs, configDir, environment string) (Provider, []string, error) { + defaultConfigDir := filepath.Join(configDir, "_default") + environmentConfigDir := filepath.Join(configDir, environment) + cfg := New() + + var configDirs []string + // Merge from least to most specific. + for _, dir := range []string{defaultConfigDir, environmentConfigDir} { + if _, err := sourceFs.Stat(dir); err == nil { + configDirs = append(configDirs, dir) + } + } + + if len(configDirs) == 0 { + return nil, nil, nil + } + + // Keep track of these so we can watch them for changes. + var dirnames []string + + for _, configDir := range configDirs { + err := afero.Walk(sourceFs, configDir, func(path string, fi os.FileInfo, err error) error { + if fi == nil || err != nil { + return nil + } + + if fi.IsDir() { + dirnames = append(dirnames, path) + return nil + } + + if !IsValidConfigFilename(path) { + return nil + } + + name := paths.Filename(filepath.Base(path)) + + item, err := metadecoders.Default.UnmarshalFileToMap(sourceFs, path) + if err != nil { + // This will be used in error reporting, use the most specific value. + dirnames = []string{path} + return errors.Wrapf(err, "failed to unmarshl config for path %q", path) + } + + var keyPath []string + + if name != "config" { + // Can be params.jp, menus.en etc. + name, lang := paths.FileAndExtNoDelimiter(name) + + keyPath = []string{name} + + if lang != "" { + keyPath = []string{"languages", lang} + switch name { + case "menu", "menus": + keyPath = append(keyPath, "menus") + case "params": + keyPath = append(keyPath, "params") + } + } + } + + root := item + if len(keyPath) > 0 { + root = make(map[string]interface{}) + m := root + for i, key := range keyPath { + if i >= len(keyPath)-1 { + m[key] = item + } else { + nm := make(map[string]interface{}) + m[key] = nm + m = nm + } + } + } + + // Migrate menu => menus etc. + RenameKeys(root) + + // Set will overwrite keys with the same name, recursively. + cfg.Set("", root) + + return nil + }) + if err != nil { + return nil, dirnames, err + } + + } + + return cfg, dirnames, nil + +} + var keyAliases maps.KeyRenamer func init() { diff --git a/hugolib/config.go b/hugolib/config.go index 091827660..cad845199 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -79,10 +79,16 @@ func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provid } if d.AbsConfigDir != "" { - dirnames, err := l.loadConfigFromConfigDir() + dcfg, dirnames, err := config.LoadConfigFromDir(l.Fs, d.AbsConfigDir, l.Environment) if err == nil { - configFiles = append(configFiles, dirnames...) + if len(dirnames) > 0 { + l.cfg.Set("", dcfg.Get("")) + configFiles = append(configFiles, dirnames...) + } } else if err != ErrNoConfigFile { + if len(dirnames) > 0 { + return nil, nil, l.wrapFileError(err, dirnames[0]) + } return nil, nil, err } } @@ -381,9 +387,9 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide hook := func(m *modules.ModulesConfig) error { for _, tc := range m.ActiveModules { - if tc.ConfigFilename() != "" { + if len(tc.ConfigFilenames()) > 0 { if tc.Watch() { - configFilenames = append(configFilenames, tc.ConfigFilename()) + configFilenames = append(configFilenames, tc.ConfigFilenames()...) } // Merge from theme config into v1 based on configured @@ -406,6 +412,7 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide HookBeforeFinalize: hook, WorkingDir: workingDir, ThemesDir: themesDir, + Environment: l.Environment, CacheDir: filecacheConfigs.CacheDirModules(), ModuleConfig: modConfig, IgnoreVendor: ignoreVendor, @@ -468,106 +475,6 @@ func (l configLoader) loadConfig(configName string) (string, error) { return filename, nil } -func (l configLoader) loadConfigFromConfigDir() ([]string, error) { - sourceFs := l.Fs - configDir := l.AbsConfigDir - - if _, err := sourceFs.Stat(configDir); err != nil { - // Config dir does not exist. - return nil, nil - } - - defaultConfigDir := filepath.Join(configDir, "_default") - environmentConfigDir := filepath.Join(configDir, l.Environment) - - var configDirs []string - // Merge from least to most specific. - for _, dir := range []string{defaultConfigDir, environmentConfigDir} { - if _, err := sourceFs.Stat(dir); err == nil { - configDirs = append(configDirs, dir) - } - } - - if len(configDirs) == 0 { - return nil, nil - } - - // Keep track of these so we can watch them for changes. - var dirnames []string - - for _, configDir := range configDirs { - err := afero.Walk(sourceFs, configDir, func(path string, fi os.FileInfo, err error) error { - if fi == nil || err != nil { - return nil - } - - if fi.IsDir() { - dirnames = append(dirnames, path) - return nil - } - - if !config.IsValidConfigFilename(path) { - return nil - } - - name := cpaths.Filename(filepath.Base(path)) - - item, err := metadecoders.Default.UnmarshalFileToMap(sourceFs, path) - if err != nil { - return l.wrapFileError(err, path) - } - - var keyPath []string - - if name != "config" { - // Can be params.jp, menus.en etc. - name, lang := cpaths.FileAndExtNoDelimiter(name) - - keyPath = []string{name} - - if lang != "" { - keyPath = []string{"languages", lang} - switch name { - case "menu", "menus": - keyPath = append(keyPath, "menus") - case "params": - keyPath = append(keyPath, "params") - } - } - } - - root := item - if len(keyPath) > 0 { - root = make(map[string]interface{}) - m := root - for i, key := range keyPath { - if i >= len(keyPath)-1 { - m[key] = item - } else { - nm := make(map[string]interface{}) - m[key] = nm - m = nm - } - } - } - - // Migrate menu => menus etc. - config.RenameKeys(root) - - // Set will overwrite keys with the same name, recursively. - l.cfg.Set("", root) - - return nil - }) - if err != nil { - return nil, err - } - - } - - return dirnames, nil -} - func (l configLoader) loadLanguageSettings(oldLangs langs.Languages) error { _, err := langs.LoadLanguageSettings(l.cfg, oldLangs) return err diff --git a/hugolib/config_test.go b/hugolib/config_test.go index 77ac9b92f..65cb246b9 100644 --- a/hugolib/config_test.go +++ b/hugolib/config_test.go @@ -318,6 +318,59 @@ name = "menu-theme" } +func TestLoadConfigFromThemeDir(t *testing.T) { + t.Parallel() + + mainConfig := ` +theme = "test-theme" + +[params] +m1 = "mv1" +` + + themeConfig := ` +[params] +t1 = "tv1" +t2 = "tv2" +` + + themeConfigDir := filepath.Join("themes", "test-theme", "config") + themeConfigDirDefault := filepath.Join(themeConfigDir, "_default") + themeConfigDirProduction := filepath.Join(themeConfigDir, "production") + + projectConfigDir := "config" + + b := newTestSitesBuilder(t) + b.WithConfigFile("toml", mainConfig).WithThemeConfigFile("toml", themeConfig) + b.Assert(b.Fs.Source.MkdirAll(themeConfigDirDefault, 0777), qt.IsNil) + b.Assert(b.Fs.Source.MkdirAll(themeConfigDirProduction, 0777), qt.IsNil) + b.Assert(b.Fs.Source.MkdirAll(projectConfigDir, 0777), qt.IsNil) + + b.WithSourceFile(filepath.Join(projectConfigDir, "config.toml"), `[params] +m2 = "mv2" +`) + b.WithSourceFile(filepath.Join(themeConfigDirDefault, "config.toml"), `[params] +t2 = "tv2d" +t3 = "tv3d" +`) + + b.WithSourceFile(filepath.Join(themeConfigDirProduction, "config.toml"), `[params] +t3 = "tv3p" +`) + + b.Build(BuildCfg{}) + + got := b.Cfg.Get("params").(maps.Params) + + b.Assert(got, qt.DeepEquals, maps.Params{ + "t3": "tv3p", + "m1": "mv1", + "t1": "tv1", + "t2": "tv2d", + }) + +} + func TestPrivacyConfig(t *testing.T) { t.Parallel() diff --git a/modules/client.go b/modules/client.go index 571ece15e..73c3242a8 100644 --- a/modules/client.go +++ b/modules/client.go @@ -653,6 +653,9 @@ type ClientConfig struct { // Absolute path to the project's themes dir. ThemesDir string + // Eg. "production" + Environment string + CacheDir string // Module cache ModuleConfig Config } diff --git a/modules/collect.go b/modules/collect.go index 163eda74a..52d75af59 100644 --- a/modules/collect.go +++ b/modules/collect.go @@ -396,17 +396,16 @@ func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error { func (c *collector) applyThemeConfig(tc *moduleAdapter) error { var ( configFilename string - cfg config.Provider themeCfg map[string]interface{} - hasConfig bool + hasConfigFile bool err error ) // Viper supports more, but this is the sub-set supported by Hugo. for _, configFormats := range config.ValidConfigFileExtensions { configFilename = filepath.Join(tc.Dir(), "config."+configFormats) - hasConfig, _ = afero.Exists(c.fs, configFilename) - if hasConfig { + hasConfigFile, _ = afero.Exists(c.fs, configFilename) + if hasConfigFile { break } } @@ -428,20 +427,38 @@ func (c *collector) applyThemeConfig(tc *moduleAdapter) error { } } - if hasConfig { + if hasConfigFile { if configFilename != "" { var err error - cfg, err = config.FromFile(c.fs, configFilename) + tc.cfg, err = config.FromFile(c.fs, configFilename) if err != nil { return errors.Wrapf(err, "failed to read module config for %q in %q", tc.Path(), configFilename) } } - tc.configFilename = configFilename - tc.cfg = cfg + tc.configFilenames = append(tc.configFilenames, configFilename) + + } + + // Also check for a config dir, which we overlay on top of the file configuration. + configDir := filepath.Join(tc.Dir(), "config") + dcfg, dirnames, err := config.LoadConfigFromDir(c.fs, configDir, c.ccfg.Environment) + if err != nil { + return err + } + + if len(dirnames) > 0 { + tc.configFilenames = append(tc.configFilenames, dirnames...) + + if hasConfigFile { + // Set will overwrite existing keys. + tc.cfg.Set("", dcfg.Get("")) + } else { + tc.cfg = dcfg + } } - config, err := decodeConfig(cfg, c.moduleConfig.replacementsMap) + config, err := decodeConfig(tc.cfg, c.moduleConfig.replacementsMap) if err != nil { return err } diff --git a/modules/module.go b/modules/module.go index a5f707635..c3343c820 100644 --- a/modules/module.go +++ b/modules/module.go @@ -30,10 +30,10 @@ type Module interface { // The decoded module config and mounts. Config() Config - // Optional configuration filename (e.g. "/themes/mytheme/config.json"). + // Optional configuration filenames (e.g. "/themes/mytheme/config.json"). // This will be added to the special configuration watch list when in // server mode. - ConfigFilename() string + ConfigFilenames() []string // Directory holding files for this module. Dir() string @@ -82,9 +82,9 @@ type moduleAdapter struct { mounts []Mount - configFilename string - cfg config.Provider - config Config + configFilenames []string + cfg config.Provider + config Config // Set if a Go module. gomod *goModule @@ -98,8 +98,8 @@ func (m *moduleAdapter) Config() Config { return m.config } -func (m *moduleAdapter) ConfigFilename() string { - return m.configFilename +func (m *moduleAdapter) ConfigFilenames() []string { + return m.configFilenames } func (m *moduleAdapter) Dir() string { -- cgit v1.2.3