diff options
-rw-r--r-- | commands/commandeer.go | 2 | ||||
-rw-r--r-- | config/allconfig/allconfig.go | 15 | ||||
-rw-r--r-- | config/allconfig/alldecoders.go | 3 | ||||
-rw-r--r-- | config/allconfig/load.go | 8 | ||||
-rw-r--r-- | config/commonConfig.go | 137 | ||||
-rw-r--r-- | config/commonConfig_test.go | 27 | ||||
-rw-r--r-- | hugofs/glob/glob.go | 25 | ||||
-rw-r--r-- | hugolib/alias.go | 2 | ||||
-rw-r--r-- | hugolib/filesystems/basefs.go | 25 | ||||
-rw-r--r-- | hugolib/hugo_sites_build_test.go | 3 | ||||
-rw-r--r-- | hugolib/site.go | 62 | ||||
-rw-r--r-- | resources/resource_cache.go | 17 |
12 files changed, 266 insertions, 60 deletions
diff --git a/commands/commandeer.go b/commands/commandeer.go index 5fb29d791..8e8522b28 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -162,6 +162,7 @@ func (r *rootCommand) ConfigFromConfig(key int32, oldConf *commonConfig) (*commo Fs: fs.Source, Filename: r.cfgFile, ConfigDir: r.cfgDir, + Logger: r.logger, Environment: r.environment, }, ) @@ -222,6 +223,7 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo Filename: r.cfgFile, ConfigDir: r.cfgDir, Environment: r.environment, + Logger: r.logger, }, ) if err != nil { diff --git a/config/allconfig/allconfig.go b/config/allconfig/allconfig.go index 4886aa561..ade7ea1be 100644 --- a/config/allconfig/allconfig.go +++ b/config/allconfig/allconfig.go @@ -27,6 +27,7 @@ import ( "time" "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/urls" "github.com/gohugoio/hugo/config" @@ -184,7 +185,7 @@ type Config struct { } type configCompiler interface { - CompileConfig() error + CompileConfig(logger loggers.Logger) error } func (c Config) cloneForLang() *Config { @@ -209,7 +210,7 @@ func (c Config) cloneForLang() *Config { return &x } -func (c *Config) CompileConfig() error { +func (c *Config) CompileConfig(logger loggers.Logger) error { var transientErr error s := c.Timeout if _, err := strconv.Atoi(s); err == nil { @@ -328,7 +329,7 @@ func (c *Config) CompileConfig() error { for _, s := range allDecoderSetups { if getCompiler := s.getCompiler; getCompiler != nil { - if err := getCompiler(c).CompileConfig(); err != nil { + if err := getCompiler(c).CompileConfig(logger); err != nil { return err } } @@ -668,8 +669,8 @@ func (c Configs) GetByLang(lang string) config.AllProvider { return nil } -// FromLoadConfigResult creates a new Config from res. -func FromLoadConfigResult(fs afero.Fs, res config.LoadConfigResult) (*Configs, error) { +// fromLoadConfigResult creates a new Config from res. +func fromLoadConfigResult(fs afero.Fs, logger loggers.Logger, res config.LoadConfigResult) (*Configs, error) { if !res.Cfg.IsSet("languages") { // We need at least one lang := res.Cfg.GetString("defaultContentLanguage") @@ -690,7 +691,7 @@ func FromLoadConfigResult(fs afero.Fs, res config.LoadConfigResult) (*Configs, e languagesConfig := cfg.GetStringMap("languages") var isMultiHost bool - if err := all.CompileConfig(); err != nil { + if err := all.CompileConfig(logger); err != nil { return nil, err } @@ -769,7 +770,7 @@ func FromLoadConfigResult(fs afero.Fs, res config.LoadConfigResult) (*Configs, e if err := decodeConfigFromParams(fs, bcfg, mergedConfig, clone, differentRootKeys); err != nil { return nil, fmt.Errorf("failed to decode config for language %q: %w", k, err) } - if err := clone.CompileConfig(); err != nil { + if err := clone.CompileConfig(logger); err != nil { return nil, err } langConfigMap[k] = clone diff --git a/config/allconfig/alldecoders.go b/config/allconfig/alldecoders.go index d7adb6e28..c6faf571d 100644 --- a/config/allconfig/alldecoders.go +++ b/config/allconfig/alldecoders.go @@ -92,6 +92,9 @@ var allDecoderSetups = map[string]decodeWeight{ p.c.Build = config.DecodeBuildConfig(p.p) return nil }, + getCompiler: func(c *Config) configCompiler { + return &c.Build + }, }, "frontmatter": { key: "frontmatter", diff --git a/config/allconfig/load.go b/config/allconfig/load.go index 6ae26d28e..ad090d60d 100644 --- a/config/allconfig/load.go +++ b/config/allconfig/load.go @@ -44,6 +44,10 @@ func LoadConfig(d ConfigSourceDescriptor) (*Configs, error) { d.Environ = os.Environ() } + if d.Logger == nil { + d.Logger = loggers.NewErrorLogger() + } + l := &configLoader{ConfigSourceDescriptor: d, cfg: config.New()} // Make sure we always do this, even in error situations, // as we have commands (e.g. "hugo mod init") that will @@ -54,7 +58,7 @@ func LoadConfig(d ConfigSourceDescriptor) (*Configs, error) { return nil, fmt.Errorf("failed to load config: %w", err) } - configs, err := FromLoadConfigResult(d.Fs, res) + configs, err := fromLoadConfigResult(d.Fs, d.Logger, res) if err != nil { return nil, fmt.Errorf("failed to create config from result: %w", err) } @@ -67,7 +71,7 @@ func LoadConfig(d ConfigSourceDescriptor) (*Configs, error) { if len(l.ModulesConfigFiles) > 0 { // Config merged in from modules. // Re-read the config. - configs, err = FromLoadConfigResult(d.Fs, res) + configs, err = fromLoadConfigResult(d.Fs, d.Logger, res) if err != nil { return nil, fmt.Errorf("failed to create config from modules config: %w", err) } diff --git a/config/commonConfig.go b/config/commonConfig.go index 8cac2e1e5..bd3e235bd 100644 --- a/config/commonConfig.go +++ b/config/commonConfig.go @@ -15,12 +15,14 @@ package config import ( "fmt" + "regexp" "sort" "strings" + "github.com/gobwas/glob" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/types" - "github.com/gobwas/glob" "github.com/gohugoio/hugo/common/herrors" "github.com/mitchellh/mapstructure" "github.com/spf13/cast" @@ -77,9 +79,29 @@ type LoadConfigResult struct { BaseConfig BaseConfig } -var DefaultBuild = BuildConfig{ +var defaultBuild = BuildConfig{ UseResourceCacheWhen: "fallback", WriteStats: false, + + CacheBusters: []CacheBuster{ + { + Source: `assets/.*\.(js|ts|jsx|tsx)`, + Target: `(js|scripts|javascript)`, + }, + { + Source: `assets/.*\.(css|sass|scss)$`, + Target: cssTargetCachebusterRe, + }, + { + Source: `(postcss|tailwind)\.config\.js`, + Target: cssTargetCachebusterRe, + }, + // This is deliberatly coarse grained; it will cache bust resources with "json" in the cache key when js files changes, which is good. + { + Source: `assets/.*\.(.*)$`, + Target: `$1`, + }, + }, } // BuildConfig holds some build related configuration. @@ -93,6 +115,14 @@ type BuildConfig struct { // Can be used to toggle off writing of the intellinsense /assets/jsconfig.js // file. NoJSConfigInAssets bool + + // Can used to control how the resource cache gets evicted on rebuilds. + CacheBusters []CacheBuster +} + +func (b BuildConfig) clone() BuildConfig { + b.CacheBusters = append([]CacheBuster{}, b.CacheBusters...) + return b } func (b BuildConfig) UseResourceCache(err error) bool { @@ -107,16 +137,47 @@ func (b BuildConfig) UseResourceCache(err error) bool { return true } +// MatchCacheBuster returns the cache buster for the given path p, nil if none. +func (s BuildConfig) MatchCacheBuster(logger loggers.Logger, p string) (func(string) bool, error) { + var matchers []func(string) bool + for _, cb := range s.CacheBusters { + if matcher := cb.compiledSource(p); matcher != nil { + matchers = append(matchers, matcher) + } + } + if len(matchers) > 0 { + return (func(cacheKey string) bool { + for _, m := range matchers { + if m(cacheKey) { + return true + } + } + return false + }), nil + } + return nil, nil +} + +func (b *BuildConfig) CompileConfig(logger loggers.Logger) error { + for i, cb := range b.CacheBusters { + if err := cb.CompileConfig(logger); err != nil { + return fmt.Errorf("failed to compile cache buster %q: %w", cb.Source, err) + } + b.CacheBusters[i] = cb + } + return nil +} + func DecodeBuildConfig(cfg Provider) BuildConfig { m := cfg.GetStringMap("build") - b := DefaultBuild + b := defaultBuild.clone() if m == nil { return b } err := mapstructure.WeakDecode(m, &b) if err != nil { - return DefaultBuild + return defaultBuild } b.UseResourceCacheWhen = strings.ToLower(b.UseResourceCacheWhen) @@ -152,7 +213,7 @@ type Server struct { compiledRedirects []glob.Glob } -func (s *Server) CompileConfig() error { +func (s *Server) CompileConfig(logger loggers.Logger) error { if s.compiledHeaders != nil { return nil } @@ -162,6 +223,7 @@ func (s *Server) CompileConfig() error { for _, r := range s.Redirects { s.compiledRedirects = append(s.compiledRedirects, glob.MustCompile(r.From)) } + return nil } @@ -228,10 +290,75 @@ type Redirect struct { Force bool } +// CacheBuster configures cache busting for assets. +type CacheBuster struct { + // Trigger for files matching this regexp. + Source string + + // Cache bust targets matching this regexp. + // This regexp can contain group matches (e.g. $1) from the source regexp. + Target string + + compiledSource func(string) func(string) bool +} + +func (c *CacheBuster) CompileConfig(logger loggers.Logger) error { + if c.compiledSource != nil { + return nil + } + source := c.Source + target := c.Target + sourceRe, err := regexp.Compile(source) + if err != nil { + return fmt.Errorf("failed to compile cache buster source %q: %w", c.Source, err) + } + var compileErr error + c.compiledSource = func(s string) func(string) bool { + m := sourceRe.FindStringSubmatch(s) + matchString := "no match" + match := m != nil + if match { + matchString = "match!" + } + logger.Debugf("cachebuster: Matching %q with source %q: %s\n", s, source, matchString) + if !match { + return nil + } + groups := m[1:] + // Replace $1, $2 etc. in target. + + for i, g := range groups { + target = strings.ReplaceAll(target, fmt.Sprintf("$%d", i+1), g) + } + targetRe, err := regexp.Compile(target) + if err != nil { + compileErr = fmt.Errorf("failed to compile cache buster target %q: %w", target, err) + return nil + } + return func(s string) bool { + match = targetRe.MatchString(s) + matchString := "no match" + if match { + matchString = "match!" + } + logger.Debugf("cachebuster: Matching %q with target %q: %s\n", s, target, matchString) + + return match + } + + } + return compileErr +} + func (r Redirect) IsZero() bool { return r.From == "" } +const ( + // Keep this a little coarse grained, some false positives are OK. + cssTargetCachebusterRe = `(css|styles|scss|sass)` +) + func DecodeServer(cfg Provider) (Server, error) { s := &Server{} diff --git a/config/commonConfig_test.go b/config/commonConfig_test.go index f05664448..106069bdc 100644 --- a/config/commonConfig_test.go +++ b/config/commonConfig_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/types" qt "github.com/frankban/quicktest" @@ -91,7 +92,7 @@ status = 301 s, err := DecodeServer(cfg) c.Assert(err, qt.IsNil) - c.Assert(s.CompileConfig(), qt.IsNil) + c.Assert(s.CompileConfig(loggers.NewErrorLogger()), qt.IsNil) c.Assert(s.MatchHeaders("/foo.jpg"), qt.DeepEquals, []types.KeyValueStr{ {Key: "X-Content-Type-Options", Value: "nosniff"}, @@ -139,3 +140,27 @@ status = 301`, } } + +func TestBuildConfigCacheBusters(t *testing.T) { + c := qt.New(t) + cfg := New() + conf := DecodeBuildConfig(cfg) + l := loggers.NewInfoLogger() + c.Assert(conf.CompileConfig(l), qt.IsNil) + + m, err := conf.MatchCacheBuster(l, "assets/foo/main.js") + c.Assert(err, qt.IsNil) + c.Assert(m, qt.IsNotNil) + c.Assert(m("scripts"), qt.IsTrue) + c.Assert(m("asdf"), qt.IsFalse) + + m, _ = conf.MatchCacheBuster(l, "tailwind.config.js") + c.Assert(m("css"), qt.IsTrue) + c.Assert(m("js"), qt.IsFalse) + + m, err = conf.MatchCacheBuster(l, "assets/foo.json") + c.Assert(err, qt.IsNil) + c.Assert(m, qt.IsNotNil) + c.Assert(m("json"), qt.IsTrue) + +} diff --git a/hugofs/glob/glob.go b/hugofs/glob/glob.go index ec9b2c7e1..dc9b4fb5b 100644 --- a/hugofs/glob/glob.go +++ b/hugofs/glob/glob.go @@ -80,6 +80,31 @@ func (gc *globCache) GetGlob(pattern string) (glob.Glob, error) { return eg.glob, eg.err } +// Or creates a new Glob from the given globs. +func Or(globs ...glob.Glob) glob.Glob { + return globSlice{globs: globs} +} + +// MatchesFunc is a convenience type to create a glob.Glob from a function. +type MatchesFunc func(s string) bool + +func (m MatchesFunc) Match(s string) bool { + return m(s) +} + +type globSlice struct { + globs []glob.Glob +} + +func (g globSlice) Match(s string) bool { + for _, g := range g.globs { + if g.Match(s) { + return true + } + } + return false +} + type globDecorator struct { // On Windows we may get filenames with Windows slashes to match, // which we need to normalize. diff --git a/hugolib/alias.go b/hugolib/alias.go index 1bc0e5424..d10f140bd 100644 --- a/hugolib/alias.go +++ b/hugolib/alias.go @@ -82,8 +82,6 @@ func (s *Site) writeDestAlias(path, permalink string, outputFormat output.Format func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, outputFormat output.Format, p page.Page) (err error) { handler := newAliasHandler(s.Tmpl(), s.Log, allowRoot) - s.Log.Debugln("creating alias:", path, "redirecting to", permalink) - targetPath, err := handler.targetPathAlias(path) if err != nil { return err diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index b90111e26..ed0d36de1 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -76,6 +76,8 @@ type BaseFs struct { theBigFs *filesystemsCollector + workingDir string + // Locks. buildMu Lockable // <project>/.hugo_build.lock } @@ -201,6 +203,27 @@ func (fs *BaseFs) ResolveJSConfigFile(name string) string { return "" } +// MakePathRelative creates a relative path from the given filename. +// It returns both the component name (e.g. layouts) and the path relative to that. +func (fs *BaseFs) MakePathRelative(filename string) (string, string) { + for _, sfs := range fs.FileSystems() { + if sfs.Contains(filename) { + if s, found := sfs.MakePathRelative(filename); found { + return sfs.Name, s + } + } + } + // May be a static file. + if s := fs.MakeStaticPathRelative(filename); s != "" { + return files.ComponentFolderStatic, s + } + // Fall back to relative to the working dir. + if strings.HasPrefix(filename, fs.workingDir) { + return "", strings.TrimPrefix(filename, fs.workingDir) + } + return "", "" +} + // SourceFilesystems contains the different source file systems. These can be // composite file systems (theme and project etc.), and they have all root // set to the source type the provides: data, i18n, static, layouts. @@ -235,6 +258,7 @@ type SourceFilesystems struct { func (s *SourceFilesystems) FileSystems() []*SourceFilesystem { return []*SourceFilesystem{ s.Content, + s.Assets, s.Data, s.I18n, s.Layouts, @@ -466,6 +490,7 @@ func NewBase(p *paths.Paths, logger loggers.Logger, options ...func(*BaseFs) err WorkDir: fs.WorkingDirReadOnly, PublishFs: publishFs, PublishFsStatic: publishFsStatic, + workingDir: p.Cfg.BaseConfig().WorkingDir, buildMu: buildMu, } diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 7b884515c..b2798c863 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -8,6 +8,7 @@ import ( "time" qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/resources/page" @@ -1393,7 +1394,7 @@ other = %q } func TestRebuildOnAssetChange(t *testing.T) { - b := newTestSitesBuilder(t).Running() + b := newTestSitesBuilder(t).Running().WithLogger(loggers.NewInfoLogger()) b.WithTemplatesAdded("index.html", ` {{ (resources.Get "data.json").Content }} `) diff --git a/hugolib/site.go b/hugolib/site.go index 301d66dac..8e220f633 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -22,7 +22,6 @@ import ( "net/url" "path" "path/filepath" - "regexp" "runtime" "sort" "strings" @@ -36,8 +35,6 @@ import ( "github.com/gohugoio/hugo/common/paths" - "github.com/gohugoio/hugo/resources" - "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/converter/hooks" @@ -45,6 +42,7 @@ import ( "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/hugofs/files" + hglob "github.com/gohugoio/hugo/hugofs/glob" "github.com/gohugoio/hugo/common/maps" @@ -483,16 +481,6 @@ func (s *Site) translateFileEvents(events []fsnotify.Event) []fsnotify.Event { return filtered } -var ( - // These are only used for cache busting, so false positives are fine. - // We also deliberately do not match for file suffixes to also catch - // directory names. - // TODO(bep) consider this when completing the relevant PR rewrite on this. - cssFileRe = regexp.MustCompile("(css|sass|scss)") - cssConfigRe = regexp.MustCompile(`(postcss|tailwind)\.config\.js`) - jsFileRe = regexp.MustCompile("(js|ts|jsx|tsx)") -) - // reBuild partially rebuilds a site given the filesystem events. // It returns whatever the content source was changed. // TODO(bep) clean up/rewrite this method. @@ -524,24 +512,16 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro logger = helpers.NewDistinctErrorLogger() ) - var cachePartitions []string - // Special case - // TODO(bep) I have a ongoing branch where I have redone the cache. Consider this there. - var ( - evictCSSRe *regexp.Regexp - evictJSRe *regexp.Regexp - ) + var cacheBusters []func(string) bool + bcfg := s.conf.Build for _, ev := range events { - if assetsFilename, _ := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" { - cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...) - if evictCSSRe == nil { - if cssFileRe.MatchString(assetsFilename) || cssConfigRe.MatchString(assetsFilename) { - evictCSSRe = cssFileRe - } - } - if evictJSRe == nil && jsFileRe.MatchString(assetsFilename) { - evictJSRe = jsFileRe + component, relFilename := s.BaseFs.MakePathRelative(ev.Name) + if relFilename != "" { + p := hglob.NormalizePath(path.Join(component, relFilename)) + g, err := bcfg.MatchCacheBuster(s.Log, p) + if err == nil && g != nil { + cacheBusters = append(cacheBusters, g) } } @@ -586,17 +566,23 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro return err } - // These in memory resource caches will be rebuilt on demand. - for _, s := range s.h.Sites { - s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...) - if evictCSSRe != nil { - s.ResourceSpec.ResourceCache.DeleteMatches(evictCSSRe) - } - if evictJSRe != nil { - s.ResourceSpec.ResourceCache.DeleteMatches(evictJSRe) + var cacheBusterOr func(string) bool + if len(cacheBusters) > 0 { + cacheBusterOr = func(s string) bool { + for _, cb := range cacheBusters { + if cb(s) { + return true + } + } + return false } } + // These in memory resource caches will be rebuilt on demand. + if len(cacheBusters) > 0 { + s.h.ResourceSpec.ResourceCache.DeleteMatches(cacheBusterOr) + } + if tmplChanged || i18nChanged { s.h.init.Reset() var prototype *deps.Deps @@ -1024,7 +1010,6 @@ func (s *Site) lookupLayouts(layouts ...string) tpl.Template { } func (s *Site) renderAndWriteXML(ctx context.Context, statCounter *uint64, name string, targetPath string, d any, templ tpl.Template) error { - s.Log.Debugf("Render XML for %q to %q", name, targetPath) renderBuffer := bp.GetBuffer() defer bp.PutBuffer(renderBuffer) @@ -1046,7 +1031,6 @@ func (s *Site) renderAndWriteXML(ctx context.Context, statCounter *uint64, name } func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath string, p *pageState, templ tpl.Template) error { - s.Log.Debugf("Render %s to %q", name, targetPath) s.h.IncrPageRender() renderBuffer := bp.GetBuffer() defer bp.PutBuffer(renderBuffer) diff --git a/resources/resource_cache.go b/resources/resource_cache.go index 8b0b363c9..388e293e8 100644 --- a/resources/resource_cache.go +++ b/resources/resource_cache.go @@ -24,7 +24,7 @@ import ( "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs/glob" + hglob "github.com/gohugoio/hugo/hugofs/glob" "github.com/gohugoio/hugo/resources/resource" @@ -83,7 +83,7 @@ var extAliasKeywords = map[string][]string{ // e.g. "scss" will also return "sass". func ResourceKeyPartitions(filename string) []string { var partitions []string - filename = glob.NormalizePath(filename) + filename = hglob.NormalizePath(filename) dir, name := path.Split(filename) ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(name)), ".") @@ -282,7 +282,7 @@ func (c *ResourceCache) DeletePartitions(partitions ...string) { } } -func (c *ResourceCache) DeleteMatches(re *regexp.Regexp) { +func (c *ResourceCache) DeleteMatchesRe(re *regexp.Regexp) { c.Lock() defer c.Unlock() @@ -292,3 +292,14 @@ func (c *ResourceCache) DeleteMatches(re *regexp.Regexp) { } } } + +func (c *ResourceCache) DeleteMatches(match func(string) bool) { + c.Lock() + defer c.Unlock() + + for k := range c.cache { + if match(k) { + delete(c.cache, k) + } + } +} |