diff options
author | Bjørn Erik Pedersen <[email protected]> | 2019-09-01 17:27:21 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <[email protected]> | 2019-09-02 16:37:28 +0200 |
commit | ce47c21a2998630f8edcbd056983d9c59a80b676 (patch) | |
tree | 01b7f680f631cc3418fa0e8457bff2b95a6f553d | |
parent | de9cbf61954201943a7b170a7d0a8b34afb5942c (diff) | |
download | hugo-ce47c21a2998630f8edcbd056983d9c59a80b676.tar.gz hugo-ce47c21a2998630f8edcbd056983d9c59a80b676.zip |
resources: Cache Exif data to disk
```bash
name old time/op new time/op delta
ImageExif/Cold_cache-4 312µs ±28% 355µs ± 7% ~ (p=0.343 n=4+4)
ImageExif/Cold_cache,_10-4 479µs ± 6% 546µs ± 0% +13.91% (p=0.029 n=4+4)
ImageExif/Warm_cache-4 272µs ± 1% 81µs ± 5% -70.30% (p=0.029 n=4+4)
name old alloc/op new alloc/op delta
ImageExif/Cold_cache-4 151kB ± 0% 161kB ± 0% +6.46% (p=0.029 n=4+4)
ImageExif/Cold_cache,_10-4 179kB ± 0% 189kB ± 0% +5.49% (p=0.029 n=4+4)
ImageExif/Warm_cache-4 151kB ± 0% 13kB ± 0% -91.52% (p=0.029 n=4+4)
name old allocs/op new allocs/op delta
ImageExif/Cold_cache-4 1.03k ± 0% 1.21k ± 0% +17.78% (p=0.029 n=4+4)
ImageExif/Cold_cache,_10-4 1.65k ± 0% 1.83k ± 0% +11.09% (p=0.029 n=4+4)
ImageExif/Warm_cache-4 1.03k ± 0% 0.28k ± 0% -72.40% (p=0.029 n=4+4)
```
Fixes #6291
-rw-r--r-- | go.mod | 4 | ||||
-rw-r--r-- | go.sum | 6 | ||||
-rw-r--r-- | hugolib/config.go | 2 | ||||
-rw-r--r-- | resources/image.go | 84 | ||||
-rw-r--r-- | resources/image_test.go | 50 | ||||
-rw-r--r-- | resources/images/exif/exif.go | 39 | ||||
-rw-r--r-- | resources/images/exif/exif_test.go | 26 |
7 files changed, 171 insertions, 40 deletions
@@ -12,6 +12,7 @@ require ( github.com/bep/debounce v1.2.0 github.com/bep/gitmap v1.1.0 github.com/bep/go-tocss v0.6.0 + github.com/bep/tmc v0.5.0 github.com/disintegration/gift v1.2.1 github.com/dustin/go-humanize v1.0.0 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 @@ -20,7 +21,7 @@ require ( github.com/fsnotify/fsnotify v1.4.7 github.com/gobwas/glob v0.2.3 github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95 - github.com/google/go-cmp v0.3.0 + github.com/google/go-cmp v0.3.1 github.com/gorilla/websocket v1.4.0 github.com/jdkato/prose v1.1.0 github.com/kyokomi/emoji v1.5.1 @@ -63,6 +64,7 @@ require ( golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 // indirect google.golang.org/appengine v1.6.0 // indirect google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.2.2 ) @@ -65,6 +65,8 @@ github.com/bep/gitmap v1.1.0 h1:vOMIdVB+2gd1VbfJPNJdLqAmn0af6NK98t4fK/GoCdA= github.com/bep/gitmap v1.1.0/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY= github.com/bep/go-tocss v0.6.0 h1:lJf+nIjsQDpifUr+NgHi9QMBnrr9cFvMvEBT+uV9Q9E= github.com/bep/go-tocss v0.6.0/go.mod h1:d9d3crzlTl+PUZLFzBUjfFCpp68K+ku10mzTlnqU/+A= +github.com/bep/tmc v0.5.0 h1:AP43LlBcCeJuXqwuQkVbTUOG6gQCo04Et4dHqOOx4hA= +github.com/bep/tmc v0.5.0/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= @@ -138,6 +140,8 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE= @@ -449,6 +453,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= diff --git a/hugolib/config.go b/hugolib/config.go index 2c46a2f32..35724ec41 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -631,7 +631,7 @@ func loadDefaultSettingsFor(v *viper.Viper) error { v.SetDefault("disableAliases", false) v.SetDefault("debug", false) v.SetDefault("disableFastRender", false) - v.SetDefault("timeout", 10000) // 10 seconds + v.SetDefault("timeout", 15000) // 15 seconds v.SetDefault("enableInlineShortcodes", false) return nil diff --git a/resources/image.go b/resources/image.go index f335e97da..8242f7119 100644 --- a/resources/image.go +++ b/resources/image.go @@ -14,15 +14,20 @@ package resources import ( + "encoding/json" "fmt" "image" "image/draw" _ "image/gif" _ "image/png" + "io" + "io/ioutil" "os" + "path" "strings" "sync" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/resources/images/exif" "github.com/gohugoio/hugo/resources/internal" @@ -55,43 +60,81 @@ type imageResource struct { // original (first). root *imageResource - exifInit sync.Once - exifInitErr error - exif *exif.Exif + metaInit sync.Once + metaInitErr error + meta *imageMeta baseResource } +type imageMeta struct { + Exif *exif.Exif +} + func (i *imageResource) Exif() (*exif.Exif, error) { return i.root.getExif() } func (i *imageResource) getExif() (*exif.Exif, error) { - i.exifInit.Do(func() { + i.metaInit.Do(func() { + supportsExif := i.Format == images.JPEG || i.Format == images.TIFF if !supportsExif { return + } - f, err := i.root.ReadSeekCloser() - if err != nil { - i.exifInitErr = err - return + key := i.getImageMetaCacheTargetPath() + + read := func(info filecache.ItemInfo, r io.Reader) error { + meta := &imageMeta{} + data, err := ioutil.ReadAll(r) + if err != nil { + return err + } + + if err = json.Unmarshal(data, &meta); err != nil { + return err + } + + i.meta = meta + + return nil } - defer f.Close() - x, err := i.getSpec().imaging.DecodeExif(f) - if err != nil { - i.exifInitErr = err - return + create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { + + f, err := i.root.ReadSeekCloser() + if err != nil { + i.metaInitErr = err + return + } + defer f.Close() + + x, err := i.getSpec().imaging.DecodeExif(f) + if err != nil { + i.metaInitErr = err + return + } + + i.meta = &imageMeta{Exif: x} + + // Also write it to cache + enc := json.NewEncoder(w) + return enc.Encode(i.meta) + } - i.exif = x + _, i.metaInitErr = i.getSpec().imageCache.fileCache.ReadOrCreate(key, read, create) }) - return i.exif, i.exifInitErr + if i.metaInitErr != nil { + return nil, i.metaInitErr + } + + return i.meta.Exif, nil } func (i *imageResource) Clone() resource.Resource { @@ -271,6 +314,17 @@ func (i *imageResource) setBasePath(conf images.ImageConfig) { i.getResourcePaths().relTargetDirFile = i.relTargetPathFromConfig(conf) } +func (i *imageResource) getImageMetaCacheTargetPath() string { + const imageMetaVersionNumber = 1 // Increment to invalidate the meta cache + + cfg := i.getSpec().imaging.Cfg + df := i.getResourcePaths().relTargetDirFile + p1, _ := helpers.FileAndExt(df.file) + h, _ := i.hash() + idStr := internal.HashString(h, i.size(), imageMetaVersionNumber, cfg) + return path.Join(df.dir, fmt.Sprintf("%s%s.json", p1, idStr)) +} + func (i *imageResource) relTargetPathFromConfig(conf images.ImageConfig) dirFile { p1, p2 := helpers.FileAndExt(i.getResourcePaths().relTargetDirFile.file) if conf.Action == "trace" { diff --git a/resources/image_test.go b/resources/image_test.go index 6d879d271..5afc4e36a 100644 --- a/resources/image_test.go +++ b/resources/image_test.go @@ -15,6 +15,7 @@ package resources import ( "fmt" + "math/big" "math/rand" "os" "path/filepath" @@ -22,6 +23,7 @@ import ( "strconv" "sync" "testing" + "time" "github.com/spf13/afero" @@ -50,6 +52,14 @@ var eq = qt.CmpEquals( cmp.Comparer(func(m1, m2 media.Type) bool { return m1.Type() == m2.Type() }), + cmp.Comparer( + func(v1, v2 *big.Rat) bool { + return v1.RatString() == v2.RatString() + }, + ), + cmp.Comparer(func(v1, v2 time.Time) bool { + return v1.Unix() == v2.Unix() + }), ) func TestImageTransformBasic(t *testing.T) { @@ -336,27 +346,35 @@ func TestSVGImageContent(t *testing.T) { func TestImageExif(t *testing.T) { c := qt.New(t) - image := fetchImage(c, "sunset.jpg") + fs := afero.NewMemMapFs() + spec := newTestResourceSpec(specDescriptor{fs: fs, c: c}) + image := fetchResourceForSpec(spec, c, "sunset.jpg").(resource.Image) - x, err := image.Exif() - c.Assert(err, qt.IsNil) - c.Assert(x, qt.Not(qt.IsNil)) + getAndCheckExif := func(c *qt.C, image resource.Image) { + x, err := image.Exif() + c.Assert(err, qt.IsNil) + c.Assert(x, qt.Not(qt.IsNil)) - c.Assert(x.Date.Format("2006-01-02"), qt.Equals, "2017-10-27") + c.Assert(x.Date.Format("2006-01-02"), qt.Equals, "2017-10-27") - // Malaga: https://goo.gl/taazZy - c.Assert(x.Lat, qt.Equals, float64(36.59744166666667)) - c.Assert(x.Long, qt.Equals, float64(-4.50846)) + // Malaga: https://goo.gl/taazZy + c.Assert(x.Lat, qt.Equals, float64(36.59744166666667)) + c.Assert(x.Long, qt.Equals, float64(-4.50846)) - v, found := x.Values["LensModel"] - c.Assert(found, qt.Equals, true) - lensModel, ok := v.(string) - c.Assert(ok, qt.Equals, true) - c.Assert(lensModel, qt.Equals, "smc PENTAX-DA* 16-50mm F2.8 ED AL [IF] SDM") + v, found := x.Tags["LensModel"] + c.Assert(found, qt.Equals, true) + lensModel, ok := v.(string) + c.Assert(ok, qt.Equals, true) + c.Assert(lensModel, qt.Equals, "smc PENTAX-DA* 16-50mm F2.8 ED AL [IF] SDM") + resized, _ := image.Resize("300x200") + x2, _ := resized.Exif() + c.Assert(x2, eq, x) + } - resized, _ := image.Resize("300x200") - x2, _ := resized.Exif() - c.Assert(x2, qt.Equals, x) + getAndCheckExif(c, image) + image = fetchResourceForSpec(spec, c, "sunset.jpg").(resource.Image) + // This will read from file cache. + getAndCheckExif(c, image) } diff --git a/resources/images/exif/exif.go b/resources/images/exif/exif.go index 7a3c982c1..b5161f770 100644 --- a/resources/images/exif/exif.go +++ b/resources/images/exif/exif.go @@ -24,6 +24,8 @@ import ( "unicode" "unicode/utf8" + "github.com/bep/tmc" + _exif "github.com/rwcarlsen/goexif/exif" "github.com/rwcarlsen/goexif/tiff" ) @@ -31,10 +33,10 @@ import ( const exifTimeLayout = "2006:01:02 15:04:05" type Exif struct { - Lat float64 - Long float64 - Date time.Time - Values map[string]interface{} + Lat float64 + Long float64 + Date time.Time + Tags Tags } type Decoder struct { @@ -139,7 +141,7 @@ func (d *Decoder) Decode(r io.Reader) (ex *Exif, err error) { return } - ex = &Exif{Lat: lat, Long: long, Date: tm, Values: walker.vals} + ex = &Exif{Lat: lat, Long: long, Date: tm, Tags: walker.vals} return } @@ -240,3 +242,30 @@ func nullString(in []byte) string { return "" } + +var tcodec *tmc.Codec + +func init() { + var err error + tcodec, err = tmc.New() + if err != nil { + panic(err) + } +} + +type Tags map[string]interface{} + +func (v *Tags) UnmarshalJSON(b []byte) error { + vv := make(map[string]interface{}) + if err := tcodec.Unmarshal(b, &vv); err != nil { + return err + } + + *v = vv + + return nil +} + +func (v Tags) MarshalJSON() ([]byte, error) { + return tcodec.Marshal(v) +} diff --git a/resources/images/exif/exif_test.go b/resources/images/exif/exif_test.go index eee60c089..c3cfad1cc 100644 --- a/resources/images/exif/exif_test.go +++ b/resources/images/exif/exif_test.go @@ -14,12 +14,15 @@ package exif import ( + "encoding/json" + "math/big" "os" "path/filepath" "testing" "time" "github.com/gohugoio/hugo/htesting/hqt" + "github.com/google/go-cmp/cmp" qt "github.com/frankban/quicktest" ) @@ -40,16 +43,24 @@ func TestExif(t *testing.T) { c.Assert(x.Lat, qt.Equals, float64(36.59744166666667)) c.Assert(x.Long, qt.Equals, float64(-4.50846)) - v, found := x.Values["LensModel"] + v, found := x.Tags["LensModel"] c.Assert(found, qt.Equals, true) lensModel, ok := v.(string) c.Assert(ok, qt.Equals, true) c.Assert(lensModel, qt.Equals, "smc PENTAX-DA* 16-50mm F2.8 ED AL [IF] SDM") - v, found = x.Values["DateTime"] + v, found = x.Tags["DateTime"] c.Assert(found, qt.Equals, true) c.Assert(v, hqt.IsSameType, time.Time{}) + // Verify that it survives a round-trip to JSON and back. + data, err := json.Marshal(x) + c.Assert(err, qt.IsNil) + x2 := &Exif{} + err = json.Unmarshal(data, x2) + + c.Assert(x2, eq, x) + } func TestExifPNG(t *testing.T) { @@ -81,3 +92,14 @@ func BenchmarkDecodeExif(b *testing.B) { f.Seek(0, 0) } } + +var eq = qt.CmpEquals( + cmp.Comparer( + func(v1, v2 *big.Rat) bool { + return v1.RatString() == v2.RatString() + }, + ), + cmp.Comparer(func(v1, v2 time.Time) bool { + return v1.Unix() == v2.Unix() + }), +) |