aboutsummaryrefslogtreecommitdiffhomepage
path: root/cache
diff options
context:
space:
mode:
authorBjørn Erik Pedersen <[email protected]>2024-05-17 17:06:47 +0200
committerBjørn Erik Pedersen <[email protected]>2024-06-04 16:07:39 +0200
commit447108fed2842e264897659856e9fd9cdc32ca23 (patch)
tree53687693e04496919dd84266cc1edc16746101b0 /cache
parentc71e24af5172e230baa5f7dfa2078721cda38df4 (diff)
downloadhugo-447108fed2842e264897659856e9fd9cdc32ca23.tar.gz
hugo-447108fed2842e264897659856e9fd9cdc32ca23.zip
Add a HTTP cache for remote resources.
Fixes #12502 Closes #11891
Diffstat (limited to 'cache')
-rw-r--r--cache/filecache/filecache.go122
-rw-r--r--cache/httpcache/httpcache.go208
-rw-r--r--cache/httpcache/httpcache_integration_test.go64
-rw-r--r--cache/httpcache/httpcache_test.go42
4 files changed, 423 insertions, 13 deletions
diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go
index 093d2941c..01c466ca6 100644
--- a/cache/filecache/filecache.go
+++ b/cache/filecache/filecache.go
@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import (
"sync"
"time"
+ "github.com/gohugoio/httpcache"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/hugofs"
@@ -182,6 +183,15 @@ func (c *Cache) ReadOrCreate(id string,
return
}
+// NamedLock locks the given id. The lock is released when the returned function is called.
+func (c *Cache) NamedLock(id string) func() {
+ id = cleanID(id)
+ c.nlocker.Lock(id)
+ return func() {
+ c.nlocker.Unlock(id)
+ }
+}
+
// GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will
// be invoked and the result cached.
// This method is protected by a named lock using the given id as identifier.
@@ -218,7 +228,23 @@ func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (It
var buff bytes.Buffer
return info,
hugio.ToReadCloser(&buff),
- afero.WriteReader(c.Fs, id, io.TeeReader(r, &buff))
+ c.writeReader(id, io.TeeReader(r, &buff))
+}
+
+func (c *Cache) writeReader(id string, r io.Reader) error {
+ dir := filepath.Dir(id)
+ if dir != "" {
+ _ = c.Fs.MkdirAll(dir, 0o777)
+ }
+ f, err := c.Fs.Create(id)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ _, _ = io.Copy(f, r)
+
+ return nil
}
// GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice.
@@ -253,9 +279,10 @@ func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (Item
return info, b, nil
}
- if err := afero.WriteReader(c.Fs, id, bytes.NewReader(b)); err != nil {
+ if err := c.writeReader(id, bytes.NewReader(b)); err != nil {
return info, nil, err
}
+
return info, b, nil
}
@@ -305,16 +332,8 @@ func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
return nil
}
- if c.maxAge > 0 {
- fi, err := c.Fs.Stat(id)
- if err != nil {
- return nil
- }
-
- if c.isExpired(fi.ModTime()) {
- c.Fs.Remove(id)
- return nil
- }
+ if removed, err := c.removeIfExpired(id); err != nil || removed {
+ return nil
}
f, err := c.Fs.Open(id)
@@ -325,6 +344,49 @@ func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
return f
}
+func (c *Cache) getBytesAndRemoveIfExpired(id string) ([]byte, bool) {
+ if c.maxAge == 0 {
+ // No caching.
+ return nil, false
+ }
+
+ f, err := c.Fs.Open(id)
+ if err != nil {
+ return nil, false
+ }
+ defer f.Close()
+
+ b, err := io.ReadAll(f)
+ if err != nil {
+ return nil, false
+ }
+
+ removed, err := c.removeIfExpired(id)
+ if err != nil {
+ return nil, false
+ }
+
+ return b, removed
+}
+
+func (c *Cache) removeIfExpired(id string) (bool, error) {
+ if c.maxAge <= 0 {
+ return false, nil
+ }
+
+ fi, err := c.Fs.Stat(id)
+ if err != nil {
+ return false, err
+ }
+
+ if c.isExpired(fi.ModTime()) {
+ c.Fs.Remove(id)
+ return true, nil
+ }
+
+ return false, nil
+}
+
func (c *Cache) isExpired(modTime time.Time) bool {
if c.maxAge < 0 {
return false
@@ -398,3 +460,37 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
func cleanID(name string) string {
return strings.TrimPrefix(filepath.Clean(name), helpers.FilePathSeparator)
}
+
+// AsHTTPCache returns an httpcache.Cache implementation for this file cache.
+// Note that none of the methods are protected by named locks, so you need to make sure
+// to do that in your own code.
+func (c *Cache) AsHTTPCache() httpcache.Cache {
+ return &httpCache{c: c}
+}
+
+type httpCache struct {
+ c *Cache
+}
+
+func (h *httpCache) Get(id string) (resp []byte, ok bool) {
+ id = cleanID(id)
+ b, removed := h.c.getBytesAndRemoveIfExpired(id)
+
+ return b, !removed
+}
+
+func (h *httpCache) Set(id string, resp []byte) {
+ if h.c.maxAge == 0 {
+ return
+ }
+
+ id = cleanID(id)
+
+ if err := h.c.writeReader(id, bytes.NewReader(resp)); err != nil {
+ panic(err)
+ }
+}
+
+func (h *httpCache) Delete(key string) {
+ h.c.Fs.Remove(key)
+}
diff --git a/cache/httpcache/httpcache.go b/cache/httpcache/httpcache.go
new file mode 100644
index 000000000..ff360001f
--- /dev/null
+++ b/cache/httpcache/httpcache.go
@@ -0,0 +1,208 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpcache
+
+import (
+ "encoding/json"
+ "time"
+
+ "github.com/gobwas/glob"
+ "github.com/gohugoio/hugo/common/predicate"
+ "github.com/gohugoio/hugo/config"
+ "github.com/mitchellh/mapstructure"
+)
+
+// DefaultConfig holds the default configuration for the HTTP cache.
+var DefaultConfig = Config{
+ Cache: Cache{
+ For: GlobMatcher{
+ Excludes: []string{"**"},
+ },
+ },
+ Polls: []PollConfig{
+ {
+ For: GlobMatcher{
+ Includes: []string{"**"},
+ },
+ Disable: true,
+ },
+ },
+}
+
+// Config holds the configuration for the HTTP cache.
+type Config struct {
+ // Configures the HTTP cache behaviour (RFC 9111).
+ // When this is not enabled for a resource, Hugo will go straight to the file cache.
+ Cache Cache
+
+ // Polls holds a list of configurations for polling remote resources to detect changes in watch mode.
+ // This can be disabled for some resources, typically if they are known to not change.
+ Polls []PollConfig
+}
+
+type Cache struct {
+ // Enable HTTP cache behaviour (RFC 9111) for these rsources.
+ For GlobMatcher
+}
+
+func (c *Config) Compile() (ConfigCompiled, error) {
+ var cc ConfigCompiled
+
+ p, err := c.Cache.For.CompilePredicate()
+ if err != nil {
+ return cc, err
+ }
+
+ cc.For = p
+
+ for _, pc := range c.Polls {
+
+ p, err := pc.For.CompilePredicate()
+ if err != nil {
+ return cc, err
+ }
+
+ cc.PollConfigs = append(cc.PollConfigs, PollConfigCompiled{
+ For: p,
+ Config: pc,
+ })
+ }
+
+ return cc, nil
+}
+
+// PollConfig holds the configuration for polling remote resources to detect changes in watch mode.
+// TODO1 make sure this enabled only in watch mode.
+type PollConfig struct {
+ // What remote resources to apply this configuration to.
+ For GlobMatcher
+
+ // Disable polling for this configuration.
+ Disable bool
+
+ // Low is the lower bound for the polling interval.
+ // This is the starting point when the resource has recently changed,
+ // if that resource stops changing, the polling interval will gradually increase towards High.
+ Low time.Duration
+
+ // High is the upper bound for the polling interval.
+ // This is the interval used when the resource is stable.
+ High time.Duration
+}
+
+func (c PollConfig) MarshalJSON() (b []byte, err error) {
+ // Marshal the durations as strings.
+ type Alias PollConfig
+ return json.Marshal(&struct {
+ Low string
+ High string
+ Alias
+ }{
+ Low: c.Low.String(),
+ High: c.High.String(),
+ Alias: (Alias)(c),
+ })
+}
+
+type GlobMatcher struct {
+ // Excludes holds a list of glob patterns that will be excluded.
+ Excludes []string
+
+ // Includes holds a list of glob patterns that will be included.
+ Includes []string
+}
+
+type ConfigCompiled struct {
+ For predicate.P[string]
+ PollConfigs []PollConfigCompiled
+}
+
+func (c *ConfigCompiled) PollConfigFor(s string) PollConfigCompiled {
+ for _, pc := range c.PollConfigs {
+ if pc.For(s) {
+ return pc
+ }
+ }
+ return PollConfigCompiled{}
+}
+
+func (c *ConfigCompiled) IsPollingDisabled() bool {
+ for _, pc := range c.PollConfigs {
+ if !pc.Config.Disable {
+ return false
+ }
+ }
+ return true
+}
+
+type PollConfigCompiled struct {
+ For predicate.P[string]
+ Config PollConfig
+}
+
+func (p PollConfigCompiled) IsZero() bool {
+ return p.For == nil
+}
+
+func (gm *GlobMatcher) CompilePredicate() (func(string) bool, error) {
+ var p predicate.P[string]
+ for _, include := range gm.Includes {
+ g, err := glob.Compile(include, '/')
+ if err != nil {
+ return nil, err
+ }
+ fn := func(s string) bool {
+ return g.Match(s)
+ }
+ p = p.Or(fn)
+ }
+
+ for _, exclude := range gm.Excludes {
+ g, err := glob.Compile(exclude, '/')
+ if err != nil {
+ return nil, err
+ }
+ fn := func(s string) bool {
+ return !g.Match(s)
+ }
+ p = p.And(fn)
+ }
+
+ return p, nil
+}
+
+func DecodeConfig(bcfg config.BaseConfig, m map[string]any) (Config, error) {
+ if len(m) == 0 {
+ return DefaultConfig, nil
+ }
+
+ var c Config
+
+ dc := &mapstructure.DecoderConfig{
+ Result: &c,
+ DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
+ WeaklyTypedInput: true,
+ }
+
+ decoder, err := mapstructure.NewDecoder(dc)
+ if err != nil {
+ return c, err
+ }
+
+ if err := decoder.Decode(m); err != nil {
+ return c, err
+ }
+
+ return c, nil
+}
diff --git a/cache/httpcache/httpcache_integration_test.go b/cache/httpcache/httpcache_integration_test.go
new file mode 100644
index 000000000..d3337c023
--- /dev/null
+++ b/cache/httpcache/httpcache_integration_test.go
@@ -0,0 +1,64 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpcache_test
+
+import (
+ "testing"
+ "time"
+
+ qt "github.com/frankban/quicktest"
+ "github.com/gohugoio/hugo/hugolib"
+)
+
+func TestConfigCustom(t *testing.T) {
+ files := `
+-- hugo.toml --
+[httpcache]
+[httpcache.cache.for]
+includes = ["**gohugo.io**"]
+[[httpcache.polls]]
+low = "5s"
+high = "32s"
+[httpcache.polls.for]
+includes = ["**gohugo.io**"]
+
+
+`
+
+ b := hugolib.Test(t, files)
+
+ httpcacheConf := b.H.Configs.Base.HTTPCache
+ compiled := b.H.Configs.Base.C.HTTPCache
+
+ b.Assert(httpcacheConf.Cache.For.Includes, qt.DeepEquals, []string{"**gohugo.io**"})
+ b.Assert(httpcacheConf.Cache.For.Excludes, qt.IsNil)
+
+ pc := compiled.PollConfigFor("https://gohugo.io/foo.jpg")
+ b.Assert(pc.Config.Low, qt.Equals, 5*time.Second)
+ b.Assert(pc.Config.High, qt.Equals, 32*time.Second)
+ b.Assert(compiled.PollConfigFor("https://example.com/foo.jpg").IsZero(), qt.IsTrue)
+}
+
+func TestConfigDefault(t *testing.T) {
+ files := `
+-- hugo.toml --
+`
+ b := hugolib.Test(t, files)
+
+ compiled := b.H.Configs.Base.C.HTTPCache
+
+ b.Assert(compiled.For("https://gohugo.io/posts.json"), qt.IsFalse)
+ b.Assert(compiled.For("https://gohugo.io/foo.jpg"), qt.IsFalse)
+ b.Assert(compiled.PollConfigFor("https://gohugo.io/foo.jpg").Config.Disable, qt.IsTrue)
+}
diff --git a/cache/httpcache/httpcache_test.go b/cache/httpcache/httpcache_test.go
new file mode 100644
index 000000000..e3659f97b
--- /dev/null
+++ b/cache/httpcache/httpcache_test.go
@@ -0,0 +1,42 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httpcache
+
+import (
+ "testing"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestGlobMatcher(t *testing.T) {
+ c := qt.New(t)
+
+ g := GlobMatcher{
+ Includes: []string{"**/*.jpg", "**.png", "**/bar/**"},
+ Excludes: []string{"**/foo.jpg", "**.css"},
+ }
+
+ p, err := g.CompilePredicate()
+ c.Assert(err, qt.IsNil)
+
+ c.Assert(p("foo.jpg"), qt.IsFalse)
+ c.Assert(p("foo.png"), qt.IsTrue)
+ c.Assert(p("foo/bar.jpg"), qt.IsTrue)
+ c.Assert(p("foo/bar.png"), qt.IsTrue)
+ c.Assert(p("foo/bar/foo.jpg"), qt.IsFalse)
+ c.Assert(p("foo/bar/foo.css"), qt.IsFalse)
+ c.Assert(p("foo.css"), qt.IsFalse)
+ c.Assert(p("foo/bar/foo.css"), qt.IsFalse)
+ c.Assert(p("foo/bar/foo.xml"), qt.IsTrue)
+}