diff options
author | Bjørn Erik Pedersen <[email protected]> | 2019-08-18 11:21:27 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <[email protected]> | 2019-08-26 15:00:44 +0200 |
commit | f9978ed16476ca6d233a89669c62c798cdf9db9d (patch) | |
tree | 02edb31008b997a3e77055060a34971fe9e8c5a4 | |
parent | 58d4c0a8be8beefbd7437b17bf7a9a381164d09b (diff) | |
download | hugo-f9978ed16476ca6d233a89669c62c798cdf9db9d.tar.gz hugo-f9978ed16476ca6d233a89669c62c798cdf9db9d.zip |
Image resource refactor
This commit pulls most of the image related logic into its own package, to make it easier to reason about and extend.
This is also a rewrite of the transformation logic used in Hugo Pipes, mostly to allow constructs like the one below:
{{ ($myimg | fingerprint ).Width }}
Fixes #5903
Fixes #6234
Fixes #6266
34 files changed, 2593 insertions, 1475 deletions
diff --git a/common/herrors/errors.go b/common/herrors/errors.go index e484ecb80..ff8eab116 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -52,6 +52,7 @@ func FprintStackTrace(w io.Writer, err error) { // defer herrors.Recover() func Recover(args ...interface{}) { if r := recover(); r != nil { + fmt.Println("ERR:", r) args = append(args, "stacktrace from panic: \n"+string(debug.Stack()), "\n") fmt.Println(args...) } diff --git a/htesting/test_helpers.go b/htesting/test_helpers.go index dc303b2e5..660c76a44 100644 --- a/htesting/test_helpers.go +++ b/htesting/test_helpers.go @@ -14,8 +14,10 @@ package htesting import ( + "math/rand" "runtime" "strings" + "time" "github.com/spf13/afero" ) @@ -37,3 +39,20 @@ func CreateTempDir(fs afero.Fs, prefix string) (string, func(), error) { } return tempDir, func() { fs.RemoveAll(tempDir) }, nil } + +// BailOut panics with a stack trace after the given duration. Useful for +// hanging tests. +func BailOut(after time.Duration) { + time.AfterFunc(after, func() { + buf := make([]byte, 1<<16) + runtime.Stack(buf, true) + panic(string(buf)) + }) + +} + +var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) + +func RandIntn(n int) int { + return rnd.Intn(n) +} diff --git a/hugolib/assets/images/sunset.jpg b/hugolib/assets/images/sunset.jpg Binary files differnew file mode 100644 index 000000000..7d7307bed --- /dev/null +++ b/hugolib/assets/images/sunset.jpg diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go index 4b3eef512..1f7addb28 100644 --- a/hugolib/pagebundler_test.go +++ b/hugolib/pagebundler_test.go @@ -42,8 +42,7 @@ import ( ) func TestPageBundlerSiteRegular(t *testing.T) { - t.Parallel() - + c := qt.New(t) baseBaseURL := "https://example.com" for _, baseURLPath := range []string{"", "/hugo"} { @@ -55,15 +54,14 @@ func TestPageBundlerSiteRegular(t *testing.T) { } ugly := ugly canonify := canonify - t.Run(fmt.Sprintf("ugly=%t,canonify=%t,path=%s", ugly, canonify, baseURLPathId), - func(t *testing.T) { - t.Parallel() + c.Run(fmt.Sprintf("ugly=%t,canonify=%t,path=%s", ugly, canonify, baseURLPathId), + func(c *qt.C) { + c.Parallel() baseURL := baseBaseURL + baseURLPath relURLBase := baseURLPath if canonify { relURLBase = "" } - c := qt.New(t) fs, cfg := newTestBundleSources(t) cfg.Set("baseURL", baseURL) cfg.Set("canonifyURLs", canonify) diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 84c871e4d..2b32587eb 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "io" "os" "path/filepath" "testing" @@ -167,6 +168,64 @@ T1: {{ $r.Content }} } +func TestResourceChainBasic(t *testing.T) { + t.Parallel() + + b := newTestSitesBuilder(t) + b.WithTemplatesAdded("index.html", ` +{{ $hello := "<h1> Hello World! </h1>" | resources.FromString "hello.html" | fingerprint "sha512" | minify | fingerprint }} + +HELLO: {{ $hello.Name }}|{{ $hello.RelPermalink }}|{{ $hello.Content | safeHTML }} + +{{ $img := resources.Get "images/sunset.jpg" }} +{{ $fit := $img.Fit "200x200" }} +{{ $fit2 := $fit.Fit "100x200" }} +{{ $img = $img | fingerprint }} +SUNSET: {{ $img.Name }}|{{ $img.RelPermalink }}|{{ $img.Width }}|{{ len $img.Content }} +FIT: {{ $fit.Name }}|{{ $fit.RelPermalink }}|{{ $fit.Width }} +`) + + fs := b.Fs.Source + + imageDir := filepath.Join("assets", "images") + b.Assert(os.MkdirAll(imageDir, 0777), qt.IsNil) + src, err := os.Open("testdata/sunset.jpg") + b.Assert(err, qt.IsNil) + out, err := fs.Create(filepath.Join(imageDir, "sunset.jpg")) + b.Assert(err, qt.IsNil) + _, err = io.Copy(out, src) + b.Assert(err, qt.IsNil) + out.Close() + + b.Running() + + for i := 0; i < 2; i++ { + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", + ` +SUNSET: images/sunset.jpg|/images/sunset.a9bf1d944e19c0f382e0d8f51de690f7d0bc8fa97390c4242a86c3e5c0737e71.jpg|900|90587 +FIT: images/sunset.jpg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x200_fit_q75_box.jpg|200 + +`) + + b.EditFiles("page1.md", ` +--- +title: "Page 1 edit" +summary: "Edited summary" +--- + +Edited content. + +`) + + b.Assert(b.Fs.Destination.Remove("public"), qt.IsNil) + b.H.ResourceSpec.ClearCaches() + + } +} + func TestResourceChain(t *testing.T) { t.Parallel() @@ -353,9 +412,11 @@ Publish 2: {{ $cssPublish2.Permalink }} "Publish 1: body{color:blue} /external1.min.css", "Publish 2: http://example.com/external2.min.css", ) - c.Assert(b.CheckExists("public/external2.min.css"), qt.Equals, true) - c.Assert(b.CheckExists("public/external1.min.css"), qt.Equals, true) - c.Assert(b.CheckExists("public/inline.min.css"), qt.Equals, false) + b.Assert(b.CheckExists("public/external2.css"), qt.Equals, false) + b.Assert(b.CheckExists("public/external1.css"), qt.Equals, false) + b.Assert(b.CheckExists("public/external2.min.css"), qt.Equals, true) + b.Assert(b.CheckExists("public/external1.min.css"), qt.Equals, true) + b.Assert(b.CheckExists("public/inline.min.css"), qt.Equals, false) }}, {"unmarshal", func() bool { return true }, func(b *sitesBuilder) { diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index e7d3b99fb..f1c19366d 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -536,6 +536,7 @@ func (s *sitesBuilder) changeEvents() []fsnotify.Event { } func (s *sitesBuilder) build(cfg BuildCfg, shouldFail bool) *sitesBuilder { + s.Helper() defer func() { s.changedFiles = nil }() diff --git a/resources/image.go b/resources/image.go index f1aae2996..e1a816942 100644 --- a/resources/image.go +++ b/resources/image.go @@ -14,198 +14,98 @@ package resources import ( - "errors" "fmt" "image" "image/color" "image/draw" - "image/jpeg" - "io" + _ "image/gif" + _ "image/png" "os" - "strconv" "strings" - "sync" "github.com/gohugoio/hugo/resources/resource" _errors "github.com/pkg/errors" "github.com/disintegration/imaging" - "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/helpers" - "github.com/mitchellh/mapstructure" + "github.com/gohugoio/hugo/resources/images" // Blind import for image.Decode - _ "image/gif" - _ "image/png" // Blind import for image.Decode _ "golang.org/x/image/webp" ) var ( - _ resource.Resource = (*Image)(nil) - _ resource.Source = (*Image)(nil) - _ resource.Cloner = (*Image)(nil) + _ resource.Image = (*imageResource)(nil) + _ resource.Source = (*imageResource)(nil) + _ resource.Cloner = (*imageResource)(nil) ) -// Imaging contains default image processing configuration. This will be fetched -// from site (or language) config. -type Imaging struct { - // Default image quality setting (1-100). Only used for JPEG images. - Quality int - - // Resample filter used. See https://github.com/disintegration/imaging - ResampleFilter string +// ImageResource represents an image resource. +type imageResource struct { + *images.Image - // The anchor used in Fill. Default is "smart", i.e. Smart Crop. - Anchor string + baseResource } -const ( - defaultJPEGQuality = 75 - defaultResampleFilter = "box" -) - -var ( - imageFormats = map[string]imaging.Format{ - ".jpg": imaging.JPEG, - ".jpeg": imaging.JPEG, - ".png": imaging.PNG, - ".tif": imaging.TIFF, - ".tiff": imaging.TIFF, - ".bmp": imaging.BMP, - ".gif": imaging.GIF, +func (i *imageResource) Clone() resource.Resource { + gr := i.baseResource.Clone().(baseResource) + return &imageResource{ + Image: i.WithSpec(gr), + baseResource: gr, } - - // Add or increment if changes to an image format's processing requires - // re-generation. - imageFormatsVersions = map[imaging.Format]int{ - imaging.PNG: 2, // Floyd Steinberg dithering - } - - // Increment to mark all processed images as stale. Only use when absolutely needed. - // See the finer grained smartCropVersionNumber and imageFormatsVersions. - mainImageVersionNumber = 0 -) - -var anchorPositions = map[string]imaging.Anchor{ - strings.ToLower("Center"): imaging.Center, - strings.ToLower("TopLeft"): imaging.TopLeft, - strings.ToLower("Top"): imaging.Top, - strings.ToLower("TopRight"): imaging.TopRight, - strings.ToLower("Left"): imaging.Left, - strings.ToLower("Right"): imaging.Right, - strings.ToLower("BottomLeft"): imaging.BottomLeft, - strings.ToLower("Bottom"): imaging.Bottom, - strings.ToLower("BottomRight"): imaging.BottomRight, } -var imageFilters = map[string]imaging.ResampleFilter{ - strings.ToLower("NearestNeighbor"): imaging.NearestNeighbor, - strings.ToLower("Box"): imaging.Box, - strings.ToLower("Linear"): imaging.Linear, - strings.ToLower("Hermite"): imaging.Hermite, - strings.ToLower("MitchellNetravali"): imaging.MitchellNetravali, - strings.ToLower("CatmullRom"): imaging.CatmullRom, - strings.ToLower("BSpline"): imaging.BSpline, - strings.ToLower("Gaussian"): imaging.Gaussian, - strings.ToLower("Lanczos"): imaging.Lanczos, - strings.ToLower("Hann"): imaging.Hann, - strings.ToLower("Hamming"): imaging.Hamming, - strings.ToLower("Blackman"): imaging.Blackman, - strings.ToLower("Bartlett"): imaging.Bartlett, - strings.ToLower("Welch"): imaging.Welch, - strings.ToLower("Cosine"): imaging.Cosine, -} - -// Image represents an image resource. -type Image struct { - config image.Config - configInit sync.Once - configLoaded bool - - imaging *Imaging - - format imaging.Format - - *genericResource -} +func (i *imageResource) cloneWithUpdates(u *transformationUpdate) (baseResource, error) { + base, err := i.baseResource.cloneWithUpdates(u) + if err != nil { + return nil, err + } -// Width returns i's width. -func (i *Image) Width() int { - i.initConfig() - return i.config.Width -} + var img *images.Image -// Height returns i's height. -func (i *Image) Height() int { - i.initConfig() - return i.config.Height -} + if u.isContenChanged() { + img = i.WithSpec(base) + } else { + img = i.Image + } -// WithNewBase implements the Cloner interface. -func (i *Image) WithNewBase(base string) resource.Resource { - return &Image{ - imaging: i.imaging, - format: i.format, - genericResource: i.genericResource.WithNewBase(base).(*genericResource)} + return &imageResource{ + Image: img, + baseResource: base, + }, nil } // Resize resizes the image to the specified width and height using the specified resampling // filter and returns the transformed image. If one of width or height is 0, the image aspect // ratio is preserved. -func (i *Image) Resize(spec string) (*Image, error) { - return i.doWithImageConfig("resize", spec, func(src image.Image, conf imageConfig) (image.Image, error) { - return imaging.Resize(src, conf.Width, conf.Height, conf.Filter), nil +func (i *imageResource) Resize(spec string) (resource.Image, error) { + return i.doWithImageConfig("resize", spec, func(src image.Image, conf images.ImageConfig) (image.Image, error) { + return i.Proc.Resize(src, conf) }) } // Fit scales down the image using the specified resample filter to fit the specified // maximum width and height. -func (i *Image) Fit(spec string) (*Image, error) { - return i.doWithImageConfig("fit", spec, func(src image.Image, conf imageConfig) (image.Image, error) { - return imaging.Fit(src, conf.Width, conf.Height, conf.Filter), nil +func (i *imageResource) Fit(spec string) (resource.Image, error) { + return i.doWithImageConfig("fit", spec, func(src image.Image, conf images.ImageConfig) (image.Image, error) { + return i.Proc.Fit(src, conf) }) } // Fill scales the image to the smallest possible size that will cover the specified dimensions, // crops the resized image to the specified dimensions using the given anchor point. // Space delimited config: 200x300 TopLeft -func (i *Image) Fill(spec string) (*Image, error) { - return i.doWithImageConfig("fill", spec, func(src image.Image, conf imageConfig) (image.Image, error) { - if conf.AnchorStr == smartCropIdentifier { - return smartCrop(src, conf.Width, conf.Height, conf.Anchor, conf.Filter) - } - return imaging.Fill(src, conf.Width, conf.Height, conf.Anchor, conf.Filter), nil +func (i *imageResource) Fill(spec string) (resource.Image, error) { + return i.doWithImageConfig("fill", spec, func(src image.Image, conf images.ImageConfig) (image.Image, error) { + return i.Proc.Fill(src, conf) }) } -// Holds configuration to create a new image from an existing one, resize etc. -type imageConfig struct { - Action string - - // Quality ranges from 1 to 100 inclusive, higher is better. - // This is only relevant for JPEG images. - // Default is 75. - Quality int - - // Rotate rotates an image by the given angle counter-clockwise. - // The rotation will be performed first. - Rotate int - - Width int - Height int - - Filter imaging.ResampleFilter - FilterStr string - - Anchor imaging.Anchor - AnchorStr string -} - -func (i *Image) isJPEG() bool { - name := strings.ToLower(i.relTargetDirFile.file) +func (i *imageResource) isJPEG() bool { + name := strings.ToLower(i.getResourcePaths().relTargetDirFile.file) return strings.HasSuffix(name, ".jpg") || strings.HasSuffix(name, ".jpeg") } @@ -218,42 +118,20 @@ const imageProcWorkers = 1 var imageProcSem = make(chan bool, imageProcWorkers) -func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, conf imageConfig) (image.Image, error)) (*Image, error) { - conf, err := parseImageConfig(spec) +func (i *imageResource) doWithImageConfig(action, spec string, f func(src image.Image, conf images.ImageConfig) (image.Image, error)) (resource.Image, error) { + conf, err := i.decodeImageConfig(action, spec) if err != nil { return nil, err } - conf.Action = action - - if conf.Quality <= 0 && i.isJPEG() { - // We need a quality setting for all JPEGs - conf.Quality = i.imaging.Quality - } - - if conf.FilterStr == "" { - conf.FilterStr = i.imaging.ResampleFilter - conf.Filter = imageFilters[conf.FilterStr] - } - - if conf.AnchorStr == "" { - conf.AnchorStr = i.imaging.Anchor - if !strings.EqualFold(conf.AnchorStr, smartCropIdentifier) { - conf.Anchor = anchorPositions[conf.AnchorStr] - } - } - return i.spec.imageCache.getOrCreate(i, conf, func() (*Image, image.Image, error) { + return i.getSpec().imageCache.getOrCreate(i, conf, func() (*imageResource, image.Image, error) { imageProcSem <- true defer func() { <-imageProcSem }() - ci := i.clone() - errOp := action - errPath := i.sourceFilename - - ci.setBasePath(conf) + errPath := i.getSourceFilename() src, err := i.decodeSource() if err != nil { @@ -267,10 +145,10 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c converted, err := f(src, conf) if err != nil { - return ci, nil, &os.PathError{Op: errOp, Path: errPath, Err: err} + return nil, nil, &os.PathError{Op: errOp, Path: errPath, Err: err} } - if i.format == imaging.PNG { + if i.Format == imaging.PNG { // Apply the colour palette from the source if paletted, ok := src.(*image.Paletted); ok { tmp := image.NewPaletted(converted.Bounds(), paletted.Palette) @@ -279,177 +157,30 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c } } - b := converted.Bounds() - ci.config = image.Config{Width: b.Max.X, Height: b.Max.Y} - ci.configLoaded = true + ci := i.clone(converted) + ci.setBasePath(conf) return ci, converted, nil }) - -} - -func (i imageConfig) key(format imaging.Format) string { - k := strconv.Itoa(i.Width) + "x" + strconv.Itoa(i.Height) - if i.Action != "" { - k += "_" + i.Action - } - if i.Quality > 0 { - k += "_q" + strconv.Itoa(i.Quality) - } - if i.Rotate != 0 { - k += "_r" + strconv.Itoa(i.Rotate) - } - anchor := i.AnchorStr - if anchor == smartCropIdentifier { - anchor = anchor + strconv.Itoa(smartCropVersionNumber) - } - - k += "_" + i.FilterStr - - if strings.EqualFold(i.Action, "fill") { - k += "_" + anchor - } - - if v, ok := imageFormatsVersions[format]; ok { - k += "_" + strconv.Itoa(v) - } - - if mainImageVersionNumber > 0 { - k += "_" + strconv.Itoa(mainImageVersionNumber) - } - - return k -} - -func newImageConfig(width, height, quality, rotate int, filter, anchor string) imageConfig { - var c imageConfig - - c.Width = width - c.Height = height - c.Quality = quality - c.Rotate = rotate - - if filter != "" { - filter = strings.ToLower(filter) - if v, ok := imageFilters[filter]; ok { - c.Filter = v - c.FilterStr = filter - } - } - - if anchor != "" { - anchor = strings.ToLower(anchor) - if v, ok := anchorPositions[anchor]; ok { - c.Anchor = v - c.AnchorStr = anchor - } - } - - return c } -func parseImageConfig(config string) (imageConfig, error) { - var ( - c imageConfig - err error - ) - - if config == "" { - return c, errors.New("image config cannot be empty") - } - - parts := strings.Fields(config) - for _, part := range parts { - part = strings.ToLower(part) - - if part == smartCropIdentifier { - c.AnchorStr = smartCropIdentifier - } else if pos, ok := anchorPositions[part]; ok { - c.Anchor = pos - c.AnchorStr = part - } else if filter, ok := imageFilters[part]; ok { - c.Filter = filter - c.FilterStr = part - } else if part[0] == 'q' { - c.Quality, err = strconv.Atoi(part[1:]) - if err != nil { - return c, err - } - if c.Quality < 1 || c.Quality > 100 { - return c, errors.New("quality ranges from 1 to 100 inclusive") - } - } else if part[0] == 'r' { - c.Rotate, err = strconv.Atoi(part[1:]) - if err != nil { - return c, err - } - } else if strings.Contains(part, "x") { - widthHeight := strings.Split(part, "x") - if len(widthHeight) <= 2 { - first := widthHeight[0] - if first != "" { - c.Width, err = strconv.Atoi(first) - if err != nil { - return c, err - } - } - - if len(widthHeight) == 2 { - second := widthHeight[1] - if second != "" { - c.Height, err = strconv.Atoi(second) - if err != nil { - return c, err - } - } - } - } else { - return c, errors.New("invalid image dimensions") - } - - } - } - - if c.Width == 0 && c.Height == 0 { - return c, errors.New("must provide Width or Height") +func (i *imageResource) decodeImageConfig(action, spec string) (images.ImageConfig, error) { + conf, err := images.DecodeImageConfig(action, spec, i.Proc.Cfg) + if err != nil { + return conf, err } - return c, nil -} - -func (i *Image) initConfig() error { - var err error - i.configInit.Do(func() { - if i.configLoaded { - return - } - - var ( - f hugio.ReadSeekCloser - config image.Config - ) - - f, err = i.ReadSeekCloser() - if err != nil { - return - } - defer f.Close() - - config, _, err = image.DecodeConfig(f) - if err != nil { - return - } - i.config = config - }) + iconf := i.Proc.Cfg - if err != nil { - return _errors.Wrap(err, "failed to load image config") + if conf.Quality <= 0 && i.isJPEG() { + // We need a quality setting for all JPEGs + conf.Quality = iconf.Quality } - return nil + return conf, nil } -func (i *Image) decodeSource() (image.Image, error) { +func (i *imageResource) decodeSource() (image.Image, error) { f, err := i.ReadSeekCloser() if err != nil { return nil, _errors.Wrap(err, "failed to open image for decode") @@ -459,80 +190,39 @@ func (i *Image) decodeSource() (image.Image, error) { return img, err } -// returns an opened file or nil if nothing to write. -func (i *Image) openDestinationsForWriting() (io.WriteCloser, error) { - targetFilenames := i.targetFilenames() - var changedFilenames []string - - // Fast path: - // This is a processed version of the original; - // check if it already existis at the destination. - for _, targetFilename := range targetFilenames { - if _, err := i.spec.BaseFs.PublishFs.Stat(targetFilename); err == nil { - continue - } - changedFilenames = append(changedFilenames, targetFilename) - } - - if len(changedFilenames) == 0 { - return nil, nil - } - - return helpers.OpenFilesForWriting(i.spec.BaseFs.PublishFs, changedFilenames...) - -} - -func (i *Image) encodeTo(conf imageConfig, img image.Image, w io.Writer) error { - switch i.format { - case imaging.JPEG: - - var rgba *image.RGBA - quality := conf.Quality +func (i *imageResource) clone(img image.Image) *imageResource { + spec := i.baseResource.Clone().(baseResource) - if nrgba, ok := img.(*image.NRGBA); ok { - if nrgba.Opaque() { - rgba = &image.RGBA{ - Pix: nrgba.Pix, - Stride: nrgba.Stride, - Rect: nrgba.Rect, - } - } - } - if rgba != nil { - return jpeg.Encode(w, rgba, &jpeg.Options{Quality: quality}) - } - return jpeg.Encode(w, img, &jpeg.Options{Quality: quality}) - default: - return imaging.Encode(w, img, i.format) + var image *images.Image + if img != nil { + image = i.WithImage(img) + } else { + image = i.WithSpec(spec) } -} -func (i *Image) clone() *Image { - g := *i.genericResource - g.resourceContent = &resourceContent{} - if g.publishOnce != nil { - g.publishOnce = &publishOnce{logger: g.publishOnce.logger} + return &imageResource{ + Image: image, + baseResource: spec, } - - return &Image{ - imaging: i.imaging, - format: i.format, - genericResource: &g} } -func (i *Image) setBasePath(conf imageConfig) { - i.relTargetDirFile = i.relTargetPathFromConfig(conf) +func (i *imageResource) setBasePath(conf images.ImageConfig) { + i.getResourcePaths().relTargetDirFile = i.relTargetPathFromConfig(conf) } -func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile { - p1, p2 := helpers.FileAndExt(i.relTargetDirFile.file) +func (i *imageResource) relTargetPathFromConfig(conf images.ImageConfig) dirFile { + p1, p2 := helpers.FileAndExt(i.getResourcePaths().relTargetDirFile.file) + if conf.Action == "trace" { + p2 = ".svg" + } - idStr := fmt.Sprintf("_hu%s_%d", i.hash, i.osFileInfo.Size()) + h, _ := i.hash() + idStr := fmt.Sprintf("_hu%s_%d", h, i.size()) // Do not change for no good reason. const md5Threshold = 100 - key := conf.key(i.format) + key := conf.Key(i.Format) // It is useful to have the key in clear text, but when nesting transforms, it // can easily be too long to read, and maybe even too long @@ -554,43 +244,7 @@ func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile { } return dirFile{ - dir: i.relTargetDirFile.dir, + dir: i.getResourcePaths().relTargetDirFile.dir, file: fmt.Sprintf("%s%s_%s%s", p1, idStr, key, p2), } - -} - -func decodeImaging(m map[string]interface{}) (Imaging, error) { - var i Imaging - if err := mapstructure.WeakDecode(m, &i); err != nil { - return i, err - } - - if i.Quality == 0 { - i.Quality = defaultJPEGQuality - } else if i.Quality < 0 || i.Quality > 100 { - return i, errors.New("JPEG quality must be a number between 1 and 100") - } - - if i.Anchor == "" || strings.EqualFold(i.Anchor, smartCropIdentifier) { - i.Anchor = smartCropIdentifier - } else { - i.Anchor = strings.ToLower(i.Anchor) - if _, found := anchorPositions[i.Anchor]; !found { - return i, errors.New("invalid anchor value in imaging config") - } - } - - if i.ResampleFilter == "" { - i.ResampleFilter = defaultResampleFilter - } else { - filter := strings.ToLower(i.ResampleFilter) - _, found := imageFilters[filter] - if !found { - return i, fmt.Errorf("%q is not a valid resample filter", filter) - } - i.ResampleFilter = filter - } - - return i, nil } diff --git a/resources/image_cache.go b/resources/image_cache.go index 3324e442e..3a9e3c2c5 100644 --- a/resources/image_cache.go +++ b/resources/image_cache.go @@ -20,7 +20,7 @@ import ( "strings" "sync" - "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/resources/images" "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/helpers" @@ -32,7 +32,7 @@ type imageCache struct { fileCache *filecache.Cache mu sync.RWMutex - store map[string]*Image + store map[string]*resourceAdapter } func (c *imageCache) isInCache(key string) bool { @@ -66,33 +66,34 @@ func (c *imageCache) normalizeKey(key string) string { func (c *imageCache) clear() { c.mu.Lock() defer c.mu.Unlock() - c.store = make(map[string]*Image) + c.store = make(map[string]*resourceAdapter) } func (c *imageCache) getOrCreate( - parent *Image, conf imageConfig, createImage func() (*Image, image.Image, error)) (*Image, error) { - + parent *imageResource, conf images.ImageConfig, + createImage func() (*imageResource, image.Image, error)) (*resourceAdapter, error) { relTarget := parent.relTargetPathFromConfig(conf) key := parent.relTargetPathForRel(relTarget.path(), false, false, false) // First check the in-memory store, then the disk. c.mu.RLock() - img, found := c.store[key] + cachedImage, found := c.store[key] c.mu.RUnlock() if found { - return img, nil + return cachedImage, nil } + var img *imageResource + // These funcs are protected by a named lock. // read clones the parent to its new name and copies // the content to the destinations. read := func(info filecache.ItemInfo, r io.Reader) error { - img = parent.clone() - img.relTargetDirFile.file = relTarget.file - img.sourceFilename = info.Name - // Make sure it's always loaded by sourceFilename. - img.openReadSeekerCloser = nil + img = parent.clone(nil) + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) w, err := img.openDestinationsForWriting() if err != nil { @@ -109,29 +110,20 @@ func (c *imageCache) getOrCreate( return err } - // create creates the image and encodes it to w (cache) and to its destinations. + // create creates the image and encodes it to the cache (w). create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) { + defer w.Close() + var conv image.Image img, conv, err = createImage() if err != nil { - w.Close() return } - img.relTargetDirFile.file = relTarget.file - img.sourceFilename = info.Name + rp := img.getResourcePaths() + rp.relTargetDirFile.file = relTarget.file + img.setSourceFilename(info.Name) - destinations, err := img.openDestinationsForWriting() - if err != nil { - w.Close() - return err - } - - if destinations != nil { - w = hugio.NewMultiWriteCloser(w, destinations) - } - defer w.Close() - - return img.encodeTo(conf, conv, w) + return img.EncodeTo(conf, conv, w) } // Now look in the file cache. @@ -147,20 +139,21 @@ func (c *imageCache) getOrCreate( } // The file is now stored in this cache. - img.sourceFs = c.fileCache.Fs + img.setSourceFs(c.fileCache.Fs) c.mu.Lock() - if img2, found := c.store[key]; found { + if cachedImage, found = c.store[key]; found { c.mu.Unlock() - return img2, nil + return cachedImage, nil } - c.store[key] = img - c.mu.Unlock() - return img, nil + imgAdapter := newResourceAdapter(parent.getSpec(), true, img) + c.store[key] = imgAdapter + c.mu.Unlock() + return imgAdapter, nil } func newImageCache(fileCache *filecache.Cache, ps *helpers.PathSpec) *imageCache { - return &imageCache{fileCache: fileCache, pathSpec: ps, store: make(map[string]*Image)} + return &imageCache{fileCache: fileCache, pathSpec: ps, store: make(map[string]*resourceAdapter)} } diff --git a/resources/image_test.go b/resources/image_test.go index 96a66d999..31169444d 100644 --- a/resources/image_test.go +++ b/resources/image_test.go @@ -18,121 +18,101 @@ import ( "math/rand" "path/filepath" "strconv" + "sync" "testing" - "github.com/gohugoio/hugo/htesting/hqt" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources/resource" - "github.com/disintegration/imaging" + "github.com/google/go-cmp/cmp" - "sync" + "github.com/gohugoio/hugo/htesting/hqt" qt "github.com/frankban/quicktest" ) -func TestParseImageConfig(t *testing.T) { - for i, this := range []struct { - in string - expect interface{} - }{ - {"300x400", newImageConfig(300, 400, 0, 0, "", "")}, - {"100x200 bottomRight", newImageConfig(100, 200, 0, 0, "", "BottomRight")}, - {"10x20 topleft Lanczos", newImageConfig(10, 20, 0, 0, "Lanczos", "topleft")}, - {"linear left 10x r180", newImageConfig(10, 0, 0, 180, "linear", "left")}, - {"x20 riGht Cosine q95", newImageConfig(0, 20, 95, 0, "cosine", "right")}, - - {"", false}, - {"foo", false}, - } { - result, err := parseImageConfig(this.in) - if b, ok := this.expect.(bool); ok && !b { - if err == nil { - t.Errorf("[%d] parseImageConfig didn't return an expected error", i) - } - } else { - if err != nil { - t.Fatalf("[%d] err: %s", i, err) - } - if fmt.Sprint(result) != fmt.Sprint(this.expect) { - t.Fatalf("[%d] got\n%v\n but expected\n%v", i, result, this.expect) - } - } - } -} +var eq = qt.CmpEquals( + cmp.Comparer(func(p1, p2 *resourceAdapter) bool { + return p1.resourceAdapterInner == p2.resourceAdapterInner + }), + cmp.Comparer(func(p1, p2 *genericResource) bool { return p1 == p2 }), + cmp.Comparer(func(m1, m2 media.Type) bool { + return m1.Type() == m2.Type() + }), +) func TestImageTransformBasic(t *testing.T) { - c := qt.New(t) image := fetchSunset(c) - fileCache := image.spec.FileCaches.ImageCache().Fs + + fileCache := image.(specProvider).getSpec().FileCaches.ImageCache().Fs + + assertWidthHeight := func(img resource.Image, w, h int) { + c.Helper() + c.Assert(img, qt.Not(qt.IsNil)) + c.Assert(img.Width(), qt.Equals, w) + c.Assert(img.Height(), qt.Equals, h) + } c.Assert(image.RelPermalink(), qt.Equals, "/a/sunset.jpg") c.Assert(image.ResourceType(), qt.Equals, "image") + assertWidthHeight(image, 900, 562) resized, err := image.Resize("300x200") c.Assert(err, qt.IsNil) c.Assert(image != resized, qt.Equals, true) - c.Assert(image.genericResource != resized.genericResource, qt.Equals, true) - c.Assert(image.sourceFilename != resized.sourceFilename, qt.Equals, true) + c.Assert(image, qt.Not(eq), resized) + assertWidthHeight(resized, 300, 200) + assertWidthHeight(image, 900, 562) resized0x, err := image.Resize("x200") c.Assert(err, qt.IsNil) - c.Assert(resized0x.Width(), qt.Equals, 320) - c.Assert(resized0x.Height(), qt.Equals, 200) - + assertWidthHeight(resized0x, 320, 200) assertFileCache(c, fileCache, resized0x.RelPermalink(), 320, 200) resizedx0, err := image.Resize("200x") c.Assert(err, qt.IsNil) - c.Assert(resizedx0.Width(), qt.Equals, 200) - c.Assert(resizedx0.Height(), qt.Equals, 125) + assertWidthHeight(resizedx0, 200, 125) assertFileCache(c, fileCache, resizedx0.RelPermalink(), 200, 125) resizedAndRotated, err := image.Resize("x200 r90") c.Assert(err, qt.IsNil) - c.Assert(resizedAndRotated.Width(), qt.Equals, 125) - c.Assert(resizedAndRotated.Height(), qt.Equals, 200) + assertWidthHeight(resizedAndRotated, 125, 200) assertFileCache(c, fileCache, resizedAndRotated.RelPermalink(), 125, 200) + assertWidthHeight(resized, 300, 200) c.Assert(resized.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_300x200_resize_q68_linear.jpg") - c.Assert(resized.Width(), qt.Equals, 300) - c.Assert(resized.Height(), qt.Equals, 200) fitted, err := resized.Fit("50x50") c.Assert(err, qt.IsNil) c.Assert(fitted.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_625708021e2bb281c9f1002f88e4753f.jpg") - c.Assert(fitted.Width(), qt.Equals, 50) - c.Assert(fitted.Height(), qt.Equals, 33) + assertWidthHeight(fitted, 50, 33) // Check the MD5 key threshold fittedAgain, _ := fitted.Fit("10x20") fittedAgain, err = fittedAgain.Fit("10x20") c.Assert(err, qt.IsNil) c.Assert(fittedAgain.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_3f65ba24dc2b7fba0f56d7f104519157.jpg") - c.Assert(fittedAgain.Width(), qt.Equals, 10) - c.Assert(fittedAgain.Height(), qt.Equals, 6) + assertWidthHeight(fittedAgain, 10, 6) filled, err := image.Fill("200x100 bottomLeft") c.Assert(err, qt.IsNil) c.Assert(filled.RelPermalink(), qt.Equals, "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_bottomleft.jpg") - c.Assert(filled.Width(), qt.Equals, 200) - c.Assert(filled.Height(), qt.Equals, 100) + assertWidthHeight(filled, 200, 100) assertFileCache(c, fileCache, filled.RelPermalink(), 200, 100) smart, err := image.Fill("200x100 smart") c.Assert(err, qt.IsNil) - c.Assert(smart.RelPermalink(), qt.Equals, fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", smartCropVersionNumber)) - c.Assert(smart.Width(), qt.Equals, 200) - c.Assert(smart.Height(), qt.Equals, 100) + c.Assert(smart.RelPermalink(), qt.Equals, fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", 1)) + assertWidthHeight(smart, 200, 100) assertFileCache(c, fileCache, smart.RelPermalink(), 200, 100) // Check cache filledAgain, err := image.Fill("200x100 bottomLeft") c.Assert(err, qt.IsNil) - c.Assert(filled == filledAgain, qt.Equals, true) - c.Assert(filled.sourceFilename == filledAgain.sourceFilename, qt.Equals, true) + c.Assert(filled, eq, filledAgain) assertFileCache(c, fileCache, filledAgain.RelPermalink(), 200, 100) - } // https://github.com/gohugoio/hugo/issues/4261 @@ -158,6 +138,7 @@ func TestImageTransformLongFilename(t *testing.T) { func TestImageTransformUppercaseExt(t *testing.T) { c := qt.New(t) image := fetchImage(c, "sunrise.JPG") + resized, err := image.Resize("200x") c.Assert(err, qt.IsNil) c.Assert(resized, qt.Not(qt.IsNil)) @@ -173,17 +154,16 @@ func TestImagePermalinkPublishOrder(t *testing.T) { } t.Run(name, func(t *testing.T) { - c := qt.New(t) spec := newTestResourceOsFs(c) - check1 := func(img *Image) { + check1 := func(img resource.Image) { resizedLink := "/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_100x50_resize_q75_box.jpg" c.Assert(img.RelPermalink(), qt.Equals, resizedLink) assertImageFile(c, spec.PublishFs, resizedLink, 100, 50) } - check2 := func(img *Image) { + check2 := func(img resource.Image) { c.Assert(img.RelPermalink(), qt.Equals, "/a/sunset.jpg") assertImageFile(c, spec.PublishFs, "a/sunset.jpg", 900, 562) } @@ -198,18 +178,16 @@ func TestImagePermalinkPublishOrder(t *testing.T) { resized, err := orignal.Resize("100x50") c.Assert(err, qt.IsNil) - check1(resized) + check1(resized.(resource.Image)) if !checkOriginalFirst { check2(orignal) } }) } - } func TestImageTransformConcurrent(t *testing.T) { - var wg sync.WaitGroup c := qt.New(t) @@ -239,12 +217,7 @@ func TestImageTransformConcurrent(t *testing.T) { t.Error(err) } - _, err = r2.decodeSource() - if err != nil { - t.Error("Err decode:", err) - } - - img = r1 + img = r2 } } }(i + 20) @@ -253,58 +226,12 @@ func TestImageTransformConcurrent(t *testing.T) { wg.Wait() } -func TestDecodeImaging(t *testing.T) { - c := qt.New(t) - m := map[string]interface{}{ - "quality": 42, - "resampleFilter": "NearestNeighbor", - "anchor": "topLeft", - } - - imaging, err := decodeImaging(m) - - c.Assert(err, qt.IsNil) - c.Assert(imaging.Quality, qt.Equals, 42) - c.Assert(imaging.ResampleFilter, qt.Equals, "nearestneighbor") - c.Assert(imaging.Anchor, qt.Equals, "topleft") - - m = map[string]interface{}{} - - imaging, err = decodeImaging(m) - c.Assert(err, qt.IsNil) - c.Assert(imaging.Quality, qt.Equals, defaultJPEGQuality) - c.Assert(imaging.ResampleFilter, qt.Equals, "box") - c.Assert(imaging.Anchor, qt.Equals, "smart") - - _, err = decodeImaging(map[string]interface{}{ - "quality": 123, - }) - c.Assert(err, qt.Not(qt.IsNil)) - - _, err = decodeImaging(map[string]interface{}{ - "resampleFilter": "asdf", - }) - c.Assert(err, qt.Not(qt.IsNil)) - - _, err = decodeImaging(map[string]interface{}{ - "anchor": "asdf", - }) - c.Assert(err, qt.Not(qt.IsNil)) - - imaging, err = decodeImaging(map[string]interface{}{ - "anchor": "Smart", - }) - c.Assert(err, qt.IsNil) - c.Assert(imaging.Anchor, qt.Equals, "smart") - -} - func TestImageWithMetadata(t *testing.T) { c := qt.New(t) image := fetchSunset(c) - var meta = []map[string]interface{}{ + meta := []map[string]interface{}{ { "title": "My Sunset", "name": "Sunset #:counter", @@ -318,71 +245,69 @@ func TestImageWithMetadata(t *testing.T) { resized, err := image.Resize("200x") c.Assert(err, qt.IsNil) c.Assert(resized.Name(), qt.Equals, "Sunset #1") - } func TestImageResize8BitPNG(t *testing.T) { - c := qt.New(t) image := fetchImage(c, "gohugoio.png") - c.Assert(image.format, qt.Equals, imaging.PNG) + c.Assert(image.MediaType().Type(), qt.Equals, "image/png") c.Assert(image.RelPermalink(), qt.Equals, "/a/gohugoio.png") c.Assert(image.ResourceType(), qt.Equals, "image") resized, err := image.Resize("800x") c.Assert(err, qt.IsNil) - c.Assert(resized.format, qt.Equals, imaging.PNG) + c.Assert(resized.MediaType().Type(), qt.Equals, "image/png") c.Assert(resized.RelPermalink(), qt.Equals, "/a/gohugoio_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_800x0_resize_linear_2.png") c.Assert(resized.Width(), qt.Equals, 800) - } func TestImageResizeInSubPath(t *testing.T) { - c := qt.New(t) image := fetchImage(c, "sub/gohugoio2.png") - fileCache := image.spec.FileCaches.ImageCache().Fs + fileCache := image.(specProvider).getSpec().FileCaches.ImageCache().Fs - c.Assert(image.format, qt.Equals, imaging.PNG) + c.Assert(image.MediaType(), eq, media.PNGType) c.Assert(image.RelPermalink(), qt.Equals, "/a/sub/gohugoio2.png") c.Assert(image.ResourceType(), qt.Equals, "image") resized, err := image.Resize("101x101") c.Assert(err, qt.IsNil) - c.Assert(resized.format, qt.Equals, imaging.PNG) + c.Assert(resized.MediaType().Type(), qt.Equals, "image/png") c.Assert(resized.RelPermalink(), qt.Equals, "/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png") c.Assert(resized.Width(), qt.Equals, 101) assertFileCache(c, fileCache, resized.RelPermalink(), 101, 101) publishedImageFilename := filepath.Clean(resized.RelPermalink()) - assertImageFile(c, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) - c.Assert(image.spec.BaseFs.PublishFs.Remove(publishedImageFilename), qt.IsNil) + + spec := image.(specProvider).getSpec() + + assertImageFile(c, spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) + c.Assert(spec.BaseFs.PublishFs.Remove(publishedImageFilename), qt.IsNil) // Cleare mem cache to simulate reading from the file cache. - resized.spec.imageCache.clear() + spec.imageCache.clear() resizedAgain, err := image.Resize("101x101") c.Assert(err, qt.IsNil) c.Assert(resizedAgain.RelPermalink(), qt.Equals, "/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png") c.Assert(resizedAgain.Width(), qt.Equals, 101) assertFileCache(c, fileCache, resizedAgain.RelPermalink(), 101, 101) - assertImageFile(c, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) - + assertImageFile(c, image.(specProvider).getSpec().BaseFs.PublishFs, publishedImageFilename, 101, 101) } func TestSVGImage(t *testing.T) { c := qt.New(t) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) svg := fetchResourceForSpec(spec, c, "circle.svg") c.Assert(svg, qt.Not(qt.IsNil)) } func TestSVGImageContent(t *testing.T) { c := qt.New(t) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) svg := fetchResourceForSpec(spec, c, "circle.svg") c.Assert(svg, qt.Not(qt.IsNil)) diff --git a/resources/images/config.go b/resources/images/config.go new file mode 100644 index 000000000..c4605c9cf --- /dev/null +++ b/resources/images/config.go @@ -0,0 +1,276 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package images + +import ( + "errors" + "fmt" + "strconv" + "strings" + + "github.com/disintegration/imaging" + "github.com/mitchellh/mapstructure" +) + +const ( + defaultJPEGQuality = 75 + defaultResampleFilter = "box" +) + +var ( + imageFormats = map[string]imaging.Format{ + ".jpg": imaging.JPEG, + ".jpeg": imaging.JPEG, + ".png": imaging.PNG, + ".tif": imaging.TIFF, + ".tiff": imaging.TIFF, + ".bmp": imaging.BMP, + ".gif": imaging.GIF, + } + + // Add or increment if changes to an image format's processing requires + // re-generation. + imageFormatsVersions = map[imaging.Format]int{ + imaging.PNG: 2, // Floyd Steinberg dithering + } + + // Increment to mark all processed images as stale. Only use when absolutely needed. + // See the finer grained smartCropVersionNumber and imageFormatsVersions. + mainImageVersionNumber = 0 + + // Increment to mark all traced SVGs as stale. + traceVersionNumber = 0 +) + +var anchorPositions = map[string]imaging.Anchor{ + strings.ToLower("Center"): imaging.Center, + strings.ToLower("TopLeft"): imaging.TopLeft, + strings.ToLower("Top"): imaging.Top, + strings.ToLower("TopRight"): imaging.TopRight, + strings.ToLower("Left"): imaging.Left, + strings.ToLower("Right"): imaging.Right, + strings.ToLower("BottomLeft"): imaging.BottomLeft, + strings.ToLower("Bottom"): imaging.Bottom, + strings.ToLower("BottomRight"): imaging.BottomRight, +} + +var imageFilters = map[string]imaging.ResampleFilter{ + strings.ToLower("NearestNeighbor"): imaging.NearestNeighbor, + strings.ToLower("Box"): imaging.Box, + strings.ToLower("Linear"): imaging.Linear, + strings.ToLower("Hermite"): imaging.Hermite, + strings.ToLower("MitchellNetravali"): imaging.MitchellNetravali, + strings.ToLower("CatmullRom"): imaging.CatmullRom, + strings.ToLower("BSpline"): imaging.BSpline, + strings.ToLower("Gaussian"): imaging.Gaussian, + strings.ToLower("Lanczos"): imaging.Lanczos, + strings.ToLower("Hann"): imaging.Hann, + strings.ToLower("Hamming"): imaging.Hamming, + strings.ToLower("Blackman"): imaging.Blackman, + strings.ToLower("Bartlett"): imaging.Bartlett, + strings.ToLower("Welch"): imaging.Welch, + strings.ToLower("Cosine"): imaging.Cosine, +} + +func ImageFormatFromExt(ext string) (imaging.Format, bool) { + f, found := imageFormats[ext] + return f, found +} + +func DecodeConfig(m map[string]interface{}) (Imaging, error) { + var i Imaging + if err := mapstructure.WeakDecode(m, &i); err != nil { + return i, err + } + + if i.Quality == 0 { + i.Quality = defaultJPEGQuality + } else if i.Quality < 0 || i.Quality > 100 { + return i, errors.New("JPEG quality must be a number between 1 and 100") + } + + if i.Anchor == "" || strings.EqualFold(i.Anchor, SmartCropIdentifier) { + i.Anchor = SmartCropIdentifier + } else { + i.Anchor = strings.ToLower(i.Anchor) + if _, found := anchorPositions[i.Anchor]; !found { + return i, errors.New("invalid anchor value in imaging config") + } + } + + if i.ResampleFilter == "" { + i.ResampleFilter = defaultResampleFilter + } else { + filter := strings.ToLower(i.ResampleFilter) + _, found := imageFilters[filter] + if !found { + return i, fmt.Errorf("%q is not a valid resample filter", filter) + } + i.ResampleFilter = filter + } + + return i, nil +} + +func DecodeImageConfig(action, config string, defaults Imaging) (ImageConfig, error) { + var ( + c ImageConfig + err error + ) + + c.Action = action + + if config == "" { + return c, errors.New("image config cannot be empty") + } + + parts := strings.Fields(config) + for _, part := range parts { + part = strings.ToLower(part) + + if part == SmartCropIdentifier { + c.AnchorStr = SmartCropIdentifier + } else if pos, ok := anchorPositions[part]; ok { + c.Anchor = pos + c.AnchorStr = part + } else if filter, ok := imageFilters[part]; ok { + c.Filter = filter + c.FilterStr = part + } else if part[0] == 'q' { + c.Quality, err = strconv.Atoi(part[1:]) + if err != nil { + return c, err + } + if c.Quality < 1 || c.Quality > 100 { + return c, errors.New("quality ranges from 1 to 100 inclusive") + } + } else if part[0] == 'r' { + c.Rotate, err = strconv.Atoi(part[1:]) + if err != nil { + return c, err + } + } else if strings.Contains(part, "x") { + widthHeight := strings.Split(part, "x") + if len(widthHeight) <= 2 { + first := widthHeight[0] + if first != "" { + c.Width, err = strconv.Atoi(first) + if err != nil { + return c, err + } + } + + if len(widthHeight) == 2 { + second := widthHeight[1] + if second != "" { + c.Height, err = strconv.Atoi(second) + if err != nil { + return c, err + } + } + } + } else { + return c, errors.New("invalid image dimensions") + } + + } + } + + if c.Width == 0 && c.Height == 0 { + return c, errors.New("must provide Width or Height") + } + + if c.FilterStr == "" { + c.FilterStr = defaults.ResampleFilter + c.Filter = imageFilters[c.FilterStr] + } + + if c.AnchorStr == "" { + c.AnchorStr = defaults.Anchor + if !strings.EqualFold(c.AnchorStr, SmartCropIdentifier) { + c.Anchor = anchorPositions[c.AnchorStr] + } + } + + return c, nil +} + +// ImageConfig holds configuration to create a new image from an existing one, resize etc. +type ImageConfig struct { + Action string + + // Quality ranges from 1 to 100 inclusive, higher is better. + // This is only relevant for JPEG images. + // Default is 75. + Quality int + + // Rotate rotates an image by the given angle counter-clockwise. + // The rotation will be performed first. + Rotate int + + Width int + Height int + + Filter imaging.ResampleFilter + FilterStr string + + Anchor imaging.Anchor + AnchorStr string +} + +func (i ImageConfig) Key(format imaging.Format) string { + k := strconv.Itoa(i.Width) + "x" + strconv.Itoa(i.Height) + if i.Action != "" { + k += "_" + i.Action + } + if i.Quality > 0 { + k += "_q" + strconv.Itoa(i.Quality) + } + if i.Rotate != 0 { + k += "_r" + strconv.Itoa(i.Rotate) + } + anchor := i.AnchorStr + if anchor == SmartCropIdentifier { + anchor = anchor + strconv.Itoa(smartCropVersionNumber) + } + + k += "_" + i.FilterStr + + if strings.EqualFold(i.Action, "fill") { + k += "_" + anchor + } + + if v, ok := imageFormatsVersions[format]; ok { + k += "_" + strconv.Itoa(v) + } + + if mainImageVersionNumber > 0 { + k += "_" + strconv.Itoa(mainImageVersionNumber) + } + + return k +} + +// Imaging contains default image processing configuration. This will be fetched +// from site (or language) config. +type Imaging struct { + // Default image quality setting (1-100). Only used for JPEG images. + Quality int + + // Resample filter used. See https://github.com/disintegration/imaging + ResampleFilter string + + // The anchor used in Fill. Default is "smart", i.e. Smart Crop. + Anchor string +} diff --git a/resources/images/config_test.go b/resources/images/config_test.go new file mode 100644 index 000000000..91f4b663a --- /dev/null +++ b/resources/images/config_test.go @@ -0,0 +1,125 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package images + +import ( + "fmt" + "strings" + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestDecodeConfig(t *testing.T) { + c := qt.New(t) + m := map[string]interface{}{ + "quality": 42, + "resampleFilter": "NearestNeighbor", + "anchor": "topLeft", + } + + imaging, err := DecodeConfig(m) + + c.Assert(err, qt.IsNil) + c.Assert(imaging.Quality, qt.Equals, 42) + c.Assert(imaging.ResampleFilter, qt.Equals, "nearestneighbor") + c.Assert(imaging.Anchor, qt.Equals, "topleft") + + m = map[string]interface{}{} + + imaging, err = DecodeConfig(m) + c.Assert(err, qt.IsNil) + c.Assert(imaging.Quality, qt.Equals, defaultJPEGQuality) + c.Assert(imaging.ResampleFilter, qt.Equals, "box") + c.Assert(imaging.Anchor, qt.Equals, "smart") + + _, err = DecodeConfig(map[string]interface{}{ + "quality": 123, + }) + c.Assert(err, qt.Not(qt.IsNil)) + + _, err = DecodeConfig(map[string]interface{}{ + "resampleFilter": "asdf", + }) + c.Assert(err, qt.Not(qt.IsNil)) + + _, err = DecodeConfig(map[string]interface{}{ + "anchor": "asdf", + }) + c.Assert(err, qt.Not(qt.IsNil)) + + imaging, err = DecodeConfig(map[string]interface{}{ + "anchor": "Smart", + }) + c.Assert(err, qt.IsNil) + c.Assert(imaging.Anchor, qt.Equals, "smart") +} + +func TestDecodeImageConfig(t *testing.T) { + for i, this := range []struct { + in string + expect interface{} + }{ + {"300x400", newImageConfig(300, 400, 0, 0, "", "")}, + {"100x200 bottomRight", newImageConfig(100, 200, 0, 0, "", "BottomRight")}, + {"10x20 topleft Lanczos", newImageConfig(10, 20, 0, 0, "Lanczos", "topleft")}, + {"linear left 10x r180", newImageConfig(10, 0, 0, 180, "linear", "left")}, + {"x20 riGht Cosine q95", newImageConfig(0, 20, 95, 0, "cosine", "right")}, + + {"", false}, + {"foo", false}, + } { + + result, err := DecodeImageConfig("resize", this.in, Imaging{}) + if b, ok := this.expect.(bool); ok && !b { + if err == nil { + t.Errorf("[%d] parseImageConfig didn't return an expected error", i) + } + } else { + if err != nil { + t.Fatalf("[%d] err: %s", i, err) + } + if fmt.Sprint(result) != fmt.Sprint(this.expect) { + t.Fatalf("[%d] got\n%v\n but expected\n%v", i, result, this.expect) + } + } + } +} + +func newImageConfig(width, height, quality, rotate int, filter, anchor string) ImageConfig { + var c ImageConfig + c.Action = "resize" + c.Width = width + c.Height = height + c.Quality = quality + c.Rotate = rotate + + if filter != "" { + filter = strings.ToLower(filter) + if v, ok := imageFilters[filter]; ok { + c.Filter = v + c.FilterStr = filter + } + } + + if anchor != "" { + anchor = strings.ToLower(anchor) + if v, ok := anchorPositions[anchor]; ok { + c.Anchor = v + c.AnchorStr = anchor + } + } + + return c +} diff --git a/resources/images/image.go b/resources/images/image.go new file mode 100644 index 000000000..b39e84972 --- /dev/null +++ b/resources/images/image.go @@ -0,0 +1,170 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package images + +import ( + "image" + "image/jpeg" + "io" + "sync" + + "github.com/disintegration/imaging" + "github.com/gohugoio/hugo/common/hugio" + "github.com/pkg/errors" +) + +func NewImage(f imaging.Format, proc *ImageProcessor, img image.Image, s Spec) *Image { + if img != nil { + return &Image{ + Format: f, + Proc: proc, + Spec: s, + imageConfig: &imageConfig{ + config: imageConfigFromImage(img), + configLoaded: true, + }, + } + } + return &Image{Format: f, Proc: proc, Spec: s, imageConfig: &imageConfig{}} +} + +type Image struct { + Format imaging.Format + + Proc *ImageProcessor + + Spec Spec + + *imageConfig +} + +func (i *Image) EncodeTo(conf ImageConfig, img image.Image, w io.Writer) error { + switch i.Format { + case imaging.JPEG: + + var rgba *image.RGBA + quality := conf.Quality + + if nrgba, ok := img.(*image.NRGBA); ok { + if nrgba.Opaque() { + rgba = &image.RGBA{ + Pix: nrgba.Pix, + Stride: nrgba.Stride, + Rect: nrgba.Rect, + } + } + } + if rgba != nil { + return jpeg.Encode(w, rgba, &jpeg.Options{Quality: quality}) + } + return jpeg.Encode(w, img, &jpeg.Options{Quality: quality}) + default: + return imaging.Encode(w, img, i.Format) + } +} + +// Height returns i's height. +func (i *Image) Height() int { + i.initConfig() + return i.config.Height +} + +// Width returns i's width. +func (i *Image) Width() int { + i.initConfig() + return i.config.Width +} + +func (i Image) WithImage(img image.Image) *Image { + i.Spec = nil + i.imageConfig = &imageConfig{ + config: imageConfigFromImage(img), + configLoaded: true, + } + + return &i +} + +func (i Image) WithSpec(s Spec) *Image { + i.Spec = s + i.imageConfig = &imageConfig{} + return &i +} + +func (i *Image) initConfig() error { + var err error + i.configInit.Do(func() { + if i.configLoaded { + return + } + + var ( + f hugio.ReadSeekCloser + config image.Config + ) + + f, err = i.Spec.ReadSeekCloser() + if err != nil { + return + } + defer f.Close() + + config, _, err = image.DecodeConfig(f) + if err != nil { + return + } + i.config = config + }) + + if err != nil { + return errors.Wrap(err, "failed to load image config") + } + + return nil +} + +type ImageProcessor struct { + Cfg Imaging +} + +func (p *ImageProcessor) Fill(src image.Image, conf ImageConfig) (image.Image, error) { + if conf.AnchorStr == SmartCropIdentifier { + return smartCrop(src, conf.Width, conf.Height, conf.Anchor, conf.Filter) + } + return imaging.Fill(src, conf.Width, conf.Height, conf.Anchor, conf.Filter), nil +} + +func (p *ImageProcessor) Fit(src image.Image, conf ImageConfig) (image.Image, error) { + return imaging.Fit(src, conf.Width, conf.Height, conf.Filter), nil +} + +func (p *ImageProcessor) Resize(src image.Image, conf ImageConfig) (image.Image, error) { + return imaging.Resize(src, conf.Width, conf.Height, conf.Filter), nil +} + +type Spec interface { + // Loads the image source. + ReadSeekCloser() (hugio.ReadSeekCloser, error) +} + +type imageConfig struct { + config image.Config + configInit sync.Once + configLoaded bool +} + +func imageConfigFromImage(img image.Image) image.Config { + b := img.Bounds() + return image.Config{Width: b.Max.X, Height: b.Max.Y} +} diff --git a/resources/smartcrop.go b/resources/images/smartcrop.go index d28a8dd03..0b35b8280 100644 --- a/resources/smartcrop.go +++ b/resources/images/smartcrop.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package resources +package images import ( "image" @@ -22,13 +22,18 @@ import ( const ( // Do not change. - smartCropIdentifier = "smart" + // TODO(bep) image unexport + SmartCropIdentifier = "smart" // This is just a increment, starting on 1. If Smart Crop improves its cropping, we // need a way to trigger a re-generation of the crops in the wild, so increment this. smartCropVersionNumber = 1 ) +func newSmartCropAnalyzer(filter imaging.ResampleFilter) smartcrop.Analyzer { + return smartcrop.NewAnalyzer(imagingResizer{filter: filter}) +} + // Needed by smartcrop type imagingResizer struct { filter imaging.ResampleFilter @@ -38,12 +43,7 @@ func (r imagingResizer) Resize(img image.Image, width, height uint) image.Image return imaging.Resize(img, int(width), int(height), r.filter) } -func newSmartCropAnalyzer(filter imaging.ResampleFilter) smartcrop.Analyzer { - return smartcrop.NewAnalyzer(imagingResizer{filter: filter}) -} - func smartCrop(img image.Image, width, height int, anchor imaging.Anchor, filter imaging.ResampleFilter) (*image.NRGBA, error) { - if width <= 0 || height <= 0 { return &image.NRGBA{}, nil } @@ -63,7 +63,6 @@ func smartCrop(img image.Image, width, height int, anchor imaging.Anchor, filter smart := newSmartCropAnalyzer(filter) rect, err := smart.FindBestCrop(img, width, height) - if err != nil { return nil, err } @@ -73,5 +72,4 @@ func smartCrop(img image.Image, width, height int, anchor imaging.Anchor, filter cropped := imaging.Crop(img, b) return imaging.Resize(cropped, width, height, filter), nil - } diff --git a/resources/internal/key.go b/resources/internal/key.go new file mode 100644 index 000000000..3dce8b350 --- /dev/null +++ b/resources/internal/key.go @@ -0,0 +1,61 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "strconv" + + bp "github.com/gohugoio/hugo/bufferpool" + + "github.com/mitchellh/hashstructure" +) + +// ResourceTransformationKey are provided by the different transformation implementations. +// It identifies the transformation (name) and its configuration (elements). +// We combine this in a chain with the rest of the transformations +// with the target filename and a content hash of the origin to use as cache key. +type ResourceTransformationKey struct { + Name string + elements []interface{} +} + +// NewResourceTransformationKey creates a new ResourceTransformationKey from the transformation +// name and elements. We will create a 64 bit FNV hash from the elements, which when combined +// with the other key elements should be unique for all practical applications. +func NewResourceTransformationKey(name string, elements ...interface{}) ResourceTransformationKey { + return ResourceTransformationKey{Name: name, elements: elements} +} + +// Value returns the Key as a string. +// Do not change this without good reasons. +func (k ResourceTransformationKey) Value() string { + if len(k.elements) == 0 { + return k.Name + } + + sb := bp.GetBuffer() + defer bp.PutBuffer(sb) + + sb.WriteString(k.Name) + for _, element := range k.elements { + hash, err := hashstructure.Hash(element, nil) + if err != nil { + panic(err) + } + sb.WriteString("_") + sb.WriteString(strconv.FormatUint(hash, 10)) + } + + return sb.String() +} diff --git a/resources/internal/key_test.go b/resources/internal/key_test.go new file mode 100644 index 000000000..9b6a23d87 --- /dev/null +++ b/resources/internal/key_test.go @@ -0,0 +1,36 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +type testStruct struct { + Name string + V1 int64 + V2 int32 + V3 int + V4 uint64 +} + +func TestResourceTransformationKey(t *testing.T) { + // We really need this key to be portable across OSes. + key := NewResourceTransformationKey("testing", + testStruct{Name: "test", V1: int64(10), V2: int32(20), V3: 30, V4: uint64(40)}) + c := qt.New(t) + c.Assert("testing_518996646957295636", qt.Equals, key.Value()) +} diff --git a/resources/resource.go b/resources/resource.go index 92bcbd0fc..3859e6044 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -17,30 +17,23 @@ import ( "fmt" "io" "io/ioutil" - "mime" "os" "path" "path/filepath" - "strings" "sync" "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/source" - "github.com/gohugoio/hugo/output" - "github.com/gohugoio/hugo/tpl" "github.com/pkg/errors" - "github.com/gohugoio/hugo/cache/filecache" - "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hugio" - "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/resource" "github.com/spf13/afero" "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/source" ) var ( @@ -51,80 +44,10 @@ var ( _ resource.Cloner = (*genericResource)(nil) _ resource.ResourcesLanguageMerger = (*resource.Resources)(nil) _ permalinker = (*genericResource)(nil) - _ collections.Slicer = (*genericResource)(nil) _ resource.Identifier = (*genericResource)(nil) + _ fileInfo = (*genericResource)(nil) ) -var noData = make(map[string]interface{}) - -type permalinker interface { - relPermalinkFor(target string) string - permalinkFor(target string) string - relTargetPathsFor(target string) []string - relTargetPaths() []string - TargetPath() string -} - -type Spec struct { - *helpers.PathSpec - - MediaTypes media.Types - OutputFormats output.Formats - - Logger *loggers.Logger - - TextTemplates tpl.TemplateParseFinder - - Permalinks page.PermalinkExpander - - // Holds default filter settings etc. - imaging *Imaging - - imageCache *imageCache - ResourceCache *ResourceCache - FileCaches filecache.Caches -} - -func NewSpec( - s *helpers.PathSpec, - fileCaches filecache.Caches, - logger *loggers.Logger, - outputFormats output.Formats, - mimeTypes media.Types) (*Spec, error) { - - imaging, err := decodeImaging(s.Cfg.GetStringMap("imaging")) - if err != nil { - return nil, err - } - - if logger == nil { - logger = loggers.NewErrorLogger() - } - - permalinks, err := page.NewPermalinkExpander(s) - if err != nil { - return nil, err - } - - rs := &Spec{PathSpec: s, - Logger: logger, - imaging: &imaging, - MediaTypes: mimeTypes, - OutputFormats: outputFormats, - Permalinks: permalinks, - FileCaches: fileCaches, - imageCache: newImageCache( - fileCaches.ImageCache(), - - s, - )} - - rs.ResourceCache = newResourceCache(rs) - - return rs, nil - -} - type ResourceSourceDescriptor struct { // TargetPaths is a callback to fetch paths's relative to its owner. TargetPaths func() page.TargetPaths @@ -161,136 +84,77 @@ func (r ResourceSourceDescriptor) Filename() string { return r.SourceFilename } -func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { - return r.newResourceFor(fd) +type ResourceTransformer interface { + resource.Resource + Transformer } -func (r *Spec) newResourceFor(fd ResourceSourceDescriptor) (resource.Resource, error) { - if fd.OpenReadSeekCloser == nil { - if fd.SourceFile != nil && fd.SourceFilename != "" { - return nil, errors.New("both SourceFile and AbsSourceFilename provided") - } else if fd.SourceFile == nil && fd.SourceFilename == "" { - return nil, errors.New("either SourceFile or AbsSourceFilename must be provided") - } - } - - if fd.RelTargetFilename == "" { - fd.RelTargetFilename = fd.Filename() - } - - if len(fd.TargetBasePaths) == 0 { - // If not set, we publish the same resource to all hosts. - fd.TargetBasePaths = r.MultihostTargetBasePaths - } - - return r.newResource(fd.Fs, fd) +type Transformer interface { + Transform(...ResourceTransformation) (ResourceTransformer, error) } -func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) { - fi := fd.FileInfo - var sourceFilename string - - if fd.OpenReadSeekCloser != nil { - } else if fd.SourceFilename != "" { - var err error - fi, err = sourceFs.Stat(fd.SourceFilename) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - sourceFilename = fd.SourceFilename - } else { - sourceFilename = fd.SourceFile.Filename() - } - - if fd.RelTargetFilename == "" { - fd.RelTargetFilename = sourceFilename - } +type baseResourceResource interface { + resource.Cloner + resource.ContentProvider + resource.Resource + resource.Identifier +} - ext := strings.ToLower(filepath.Ext(fd.RelTargetFilename)) - mimeType, found := r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, ".")) - // TODO(bep) we need to handle these ambigous types better, but in this context - // we most likely want the application/xml type. - if mimeType.Suffix() == "xml" && mimeType.SubType == "rss" { - mimeType, found = r.MediaTypes.GetByType("application/xml") - } +type baseResourceInternal interface { + resource.Source - if !found { - // A fallback. Note that mime.TypeByExtension is slow by Hugo standards, - // so we should configure media types to avoid this lookup for most - // situations. - mimeStr := mime.TypeByExtension(ext) - if mimeStr != "" { - mimeType, _ = media.FromStringAndExt(mimeStr, ext) - } - } + fileInfo + metaAssigner + targetPather - gr := r.newGenericResourceWithBase( - sourceFs, - fd.LazyPublish, - fd.OpenReadSeekCloser, - fd.TargetBasePaths, - fd.TargetPaths, - fi, - sourceFilename, - fd.RelTargetFilename, - mimeType) - - if mimeType.MainType == "image" { - imgFormat, ok := imageFormats[ext] - if !ok { - // This allows SVG etc. to be used as resources. They will not have the methods of the Image, but - // that would not (currently) have worked. - return gr, nil - } + ReadSeekCloser() (hugio.ReadSeekCloser, error) - if err := gr.initHash(); err != nil { - return nil, err - } + // Internal + cloneWithUpdates(*transformationUpdate) (baseResource, error) + tryTransformedFileCache(key string, u *transformationUpdate) io.ReadCloser - return &Image{ - format: imgFormat, - imaging: r.imaging, - genericResource: gr}, nil - } - return gr, nil + specProvider + getResourcePaths() *resourcePathDescriptor + getTargetFilenames() []string + openDestinationsForWriting() (io.WriteCloser, error) + openPublishFileForWriting(relTargetPath string) (io.WriteCloser, error) + relTargetPathForRel(rel string, addBaseTargetPath, isAbs, isURL bool) string } -// TODO(bep) unify -func (r *Spec) IsInImageCache(key string) bool { - // This is used for cache pruning. We currently only have images, but we could - // imagine expanding on this. - return r.imageCache.isInCache(key) +type specProvider interface { + getSpec() *Spec } -func (r *Spec) DeleteCacheByPrefix(prefix string) { - r.imageCache.deleteByPrefix(prefix) +type baseResource interface { + baseResourceResource + baseResourceInternal } -func (r *Spec) ClearCaches() { - r.imageCache.clear() - r.ResourceCache.clear() +type commonResource struct { } -func (r *Spec) CacheStats() string { - r.imageCache.mu.RLock() - defer r.imageCache.mu.RUnlock() - - s := fmt.Sprintf("Cache entries: %d", len(r.imageCache.store)) - - count := 0 - for k := range r.imageCache.store { - if count > 5 { - break +// Slice is not meant to be used externally. It's a bridge function +// for the template functions. See collections.Slice. +func (commonResource) Slice(in interface{}) (interface{}, error) { + switch items := in.(type) { + case resource.Resources: + return items, nil + case []interface{}: + groups := make(resource.Resources, len(items)) + for i, v := range items { + g, ok := v.(resource.Resource) + if !ok { + return nil, fmt.Errorf("type %T is not a Resource", v) + } + groups[i] = g + { + } } - s += "\n" + k - count++ + return groups, nil + default: + return nil, fmt.Errorf("invalid slice type %T", items) } - - return s } type dirFile struct { @@ -304,91 +168,33 @@ func (d dirFile) path() string { return path.Join(d.dir, d.file) } -type resourcePathDescriptor struct { - // The relative target directory and filename. - relTargetDirFile dirFile - - // Callback used to construct a target path relative to its owner. - targetPathBuilder func() page.TargetPaths - - // This will normally be the same as above, but this will only apply to publishing - // of resources. It may be mulltiple values when in multihost mode. - baseTargetPathDirs []string - - // baseOffset is set when the output format's path has a offset, e.g. for AMP. - baseOffset string -} - -type resourceContent struct { - content string - contentInit sync.Once -} - -type resourceHash struct { - hash string - hashInit sync.Once -} - -type publishOnce struct { - publisherInit sync.Once - publisherErr error - logger *loggers.Logger -} - -func (l *publishOnce) publish(s resource.Source) error { - l.publisherInit.Do(func() { - l.publisherErr = s.Publish() - if l.publisherErr != nil { - l.logger.ERROR.Printf("failed to publish Resource: %s", l.publisherErr) - } - }) - return l.publisherErr +type fileInfo interface { + getSourceFilename() string + setSourceFilename(string) + setSourceFs(afero.Fs) + hash() (string, error) + size() int } // genericResource represents a generic linkable resource. type genericResource struct { - commonResource - resourcePathDescriptor + *resourcePathDescriptor + *resourceFileInfo + *resourceContent + + spec *Spec title string name string params map[string]interface{} - - // Absolute filename to the source, including any content folder path. - // Note that this is absolute in relation to the filesystem it is stored in. - // It can be a base path filesystem, and then this filename will not match - // the path to the file on the real filesystem. - sourceFilename string - - // Will be set if this resource is backed by something other than a file. - openReadSeekerCloser resource.OpenReadSeekCloser - - // A hash of the source content. Is only calculated in caching situations. - *resourceHash - - // This may be set to tell us to look in another filesystem for this resource. - // We, by default, use the sourceFs filesystem in the spec below. - sourceFs afero.Fs - - spec *Spec + data map[string]interface{} resourceType string mediaType media.Type - - osFileInfo os.FileInfo - - // We create copies of this struct, so this needs to be a pointer. - *resourceContent - - // May be set to signal lazy/delayed publishing. - *publishOnce } -type commonResource struct { -} - -func (l *genericResource) Data() interface{} { - return noData +func (l *genericResource) Clone() resource.Resource { + return l.clone() } func (l *genericResource) Content() (interface{}, error) { @@ -399,72 +205,80 @@ func (l *genericResource) Content() (interface{}, error) { return l.content, nil } -func (l *genericResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) { - if l.openReadSeekerCloser != nil { - return l.openReadSeekerCloser() +func (l *genericResource) Data() interface{} { + return l.data +} + +func (l *genericResource) Key() string { + return l.relTargetDirFile.path() +} + +func (l *genericResource) MediaType() media.Type { + return l.mediaType +} + +func (l *genericResource) Name() string { + return l.name +} + +func (l *genericResource) Params() map[string]interface{} { + return l.params +} + +func (l *genericResource) Permalink() string { + return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetDirFile.path(), true), l.spec.BaseURL.HostURL()) +} + +func (l *genericResource) Publish() error { + fr, err := l.ReadSeekCloser() + if err != nil { + return err } + defer fr.Close() - f, err := l.getSourceFs().Open(l.sourceFilename) + fw, err := helpers.OpenFilesForWriting(l.spec.BaseFs.PublishFs, l.getTargetFilenames()...) if err != nil { - return nil, err + return err } - return f, nil + defer fw.Close() + _, err = io.Copy(fw, fr) + return err } -func (l *genericResource) MediaType() media.Type { - return l.mediaType +func (l *genericResource) RelPermalink() string { + return l.relPermalinkFor(l.relTargetDirFile.path()) } -// Implement the Cloner interface. -func (l genericResource) WithNewBase(base string) resource.Resource { - l.baseOffset = base - l.resourceContent = &resourceContent{} - return &l +func (l *genericResource) ResourceType() string { + return l.resourceType } -// Slice is not meant to be used externally. It's a bridge function -// for the template functions. See collections.Slice. -func (commonResource) Slice(in interface{}) (interface{}, error) { - switch items := in.(type) { - case resource.Resources: - return items, nil - case []interface{}: - groups := make(resource.Resources, len(items)) - for i, v := range items { - g, ok := v.(resource.Resource) - if !ok { - return nil, fmt.Errorf("type %T is not a Resource", v) - } - groups[i] = g - } - return groups, nil - default: - return nil, fmt.Errorf("invalid slice type %T", items) - } +func (l *genericResource) String() string { + return fmt.Sprintf("Resource(%s: %s)", l.resourceType, l.name) } -func (l *genericResource) initHash() error { - var err error - l.hashInit.Do(func() { - var hash string - var f hugio.ReadSeekCloser - f, err = l.ReadSeekCloser() - if err != nil { - err = errors.Wrap(err, "failed to open source file") - return - } - defer f.Close() +// Path is stored with Unix style slashes. +func (l *genericResource) TargetPath() string { + return l.relTargetDirFile.path() +} - hash, err = helpers.MD5FromFileFast(f) - if err != nil { - return - } - l.hash = hash +func (l *genericResource) Title() string { + return l.title +} - }) +func (l *genericResource) createBasePath(rel string, isURL bool) string { + if l.targetPathBuilder == nil { + return rel + } + tp := l.targetPathBuilder() - return err + if isURL { + return path.Join(tp.SubResourceBaseLink, rel) + } + + // TODO(bep) path + return path.Join(filepath.ToSlash(tp.SubResourceBaseTarget), rel) } func (l *genericResource) initContent() error { @@ -484,100 +298,141 @@ func (l *genericResource) initContent() error { } l.content = string(b) - }) return err } -func (l *genericResource) getSourceFs() afero.Fs { - return l.sourceFs +func (l *genericResource) setName(name string) { + l.name = name } -func (l *genericResource) publishIfNeeded() { - if l.publishOnce != nil { - l.publishOnce.publish(l) - } +func (l *genericResource) getResourcePaths() *resourcePathDescriptor { + return l.resourcePathDescriptor } -func (l *genericResource) Permalink() string { - l.publishIfNeeded() - return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetDirFile.path(), true), l.spec.BaseURL.HostURL()) +func (l *genericResource) getSpec() *Spec { + return l.spec } -func (l *genericResource) RelPermalink() string { - l.publishIfNeeded() - return l.relPermalinkFor(l.relTargetDirFile.path()) +func (l *genericResource) getTargetFilenames() []string { + paths := l.relTargetPaths() + for i, p := range paths { + paths[i] = filepath.Clean(p) + } + return paths } -func (l *genericResource) Key() string { - return l.relTargetDirFile.path() +func (l *genericResource) setTitle(title string) { + l.title = title } -func (l *genericResource) relPermalinkFor(target string) string { - return l.relPermalinkForRel(target, false) - +func (r *genericResource) tryTransformedFileCache(key string, u *transformationUpdate) io.ReadCloser { + fi, f, meta, found := r.spec.ResourceCache.getFromFile(key) + if !found { + return nil + } + u.sourceFilename = &fi.Name + mt, _ := r.spec.MediaTypes.GetByType(meta.MediaTypeV) + u.mediaType = mt + u.data = meta.MetaData + u.targetPath = meta.Target + return f } -func (l *genericResource) permalinkFor(target string) string { - return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(target, true), l.spec.BaseURL.HostURL()) -} -func (l *genericResource) relTargetPathsFor(target string) []string { - return l.relTargetPathsForRel(target) +func (r *genericResource) mergeData(in map[string]interface{}) { + if len(in) == 0 { + return + } + if r.data == nil { + r.data = make(map[string]interface{}) + } + for k, v := range in { + if _, found := r.data[k]; !found { + r.data[k] = v + } + } } -func (l *genericResource) relTargetPaths() []string { - return l.relTargetPathsForRel(l.TargetPath()) -} +func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResource, error) { + r := rc.clone() -func (l *genericResource) Name() string { - return l.name -} + if u.content != nil { + r.contentInit.Do(func() { + r.content = *u.content + r.openReadSeekerCloser = func() (hugio.ReadSeekCloser, error) { + return hugio.NewReadSeekerNoOpCloserFromString(r.content), nil + } + }) + } -func (l *genericResource) Title() string { - return l.title -} + r.mediaType = u.mediaType -func (l *genericResource) Params() map[string]interface{} { - return l.params -} + if u.sourceFilename != nil { + r.setSourceFilename(*u.sourceFilename) + } -func (l *genericResource) setTitle(title string) { - l.title = title + if u.sourceFs != nil { + r.setSourceFs(u.sourceFs) + } + + if u.targetPath == "" { + return nil, errors.New("missing targetPath") + } + + fpath, fname := path.Split(u.targetPath) + r.resourcePathDescriptor.relTargetDirFile = dirFile{dir: fpath, file: fname} + + r.mergeData(u.data) + + return r, nil } -func (l *genericResource) setName(name string) { - l.name = name +func (l genericResource) clone() *genericResource { + gi := *l.resourceFileInfo + rp := *l.resourcePathDescriptor + l.resourceFileInfo = &gi + l.resourcePathDescriptor = &rp + l.resourceContent = &resourceContent{} + return &l } -func (l *genericResource) updateParams(params map[string]interface{}) { - if l.params == nil { - l.params = params - return - } +// returns an opened file or nil if nothing to write. +func (l *genericResource) openDestinationsForWriting() (io.WriteCloser, error) { + targetFilenames := l.getTargetFilenames() + var changedFilenames []string - // Sets the params not already set - for k, v := range params { - if _, found := l.params[k]; !found { - l.params[k] = v + // Fast path: + // This is a processed version of the original; + // check if it already existis at the destination. + for _, targetFilename := range targetFilenames { + if _, err := l.getSpec().BaseFs.PublishFs.Stat(targetFilename); err == nil { + continue } + changedFilenames = append(changedFilenames, targetFilename) } + + if len(changedFilenames) == 0 { + return nil, nil + } + + return helpers.OpenFilesForWriting(l.getSpec().BaseFs.PublishFs, changedFilenames...) } -func (l *genericResource) relPermalinkForRel(rel string, isAbs bool) string { - return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, false, isAbs, true)) +func (r *genericResource) openPublishFileForWriting(relTargetPath string) (io.WriteCloser, error) { + return helpers.OpenFilesForWriting(r.spec.BaseFs.PublishFs, r.relTargetPathsFor(relTargetPath)...) } -func (l *genericResource) relTargetPathsForRel(rel string) []string { - if len(l.baseTargetPathDirs) == 0 { - return []string{l.relTargetPathForRelAndBasePath(rel, "", false, false)} - } +func (l *genericResource) permalinkFor(target string) string { + return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(target, true), l.spec.BaseURL.HostURL()) +} - var targetPaths = make([]string, len(l.baseTargetPathDirs)) - for i, dir := range l.baseTargetPathDirs { - targetPaths[i] = l.relTargetPathForRelAndBasePath(rel, dir, false, false) - } - return targetPaths +func (l *genericResource) relPermalinkFor(target string) string { + return l.relPermalinkForRel(target, false) +} + +func (l *genericResource) relPermalinkForRel(rel string, isAbs bool) string { + return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, false, isAbs, true)) } func (l *genericResource) relTargetPathForRel(rel string, addBaseTargetPath, isAbs, isURL bool) string { @@ -592,20 +447,6 @@ func (l *genericResource) relTargetPathForRel(rel string, addBaseTargetPath, isA return l.relTargetPathForRelAndBasePath(rel, basePath, isAbs, isURL) } -func (l *genericResource) createBasePath(rel string, isURL bool) string { - if l.targetPathBuilder == nil { - return rel - } - tp := l.targetPathBuilder() - - if isURL { - return path.Join(tp.SubResourceBaseLink, rel) - } - - // TODO(bep) path - return path.Join(filepath.ToSlash(tp.SubResourceBaseTarget), rel) -} - func (l *genericResource) relTargetPathForRelAndBasePath(rel, basePath string, isAbs, isURL bool) string { rel = l.createBasePath(rel, isURL) @@ -631,117 +472,153 @@ func (l *genericResource) relTargetPathForRelAndBasePath(rel, basePath string, i return rel } -func (l *genericResource) ResourceType() string { - return l.resourceType +func (l *genericResource) relTargetPaths() []string { + return l.relTargetPathsForRel(l.TargetPath()) } -func (l *genericResource) String() string { - return fmt.Sprintf("Resource(%s: %s)", l.resourceType, l.name) +func (l *genericResource) relTargetPathsFor(target string) []string { + return l.relTargetPathsForRel(target) } -func (l *genericResource) Publish() error { - fr, err := l.ReadSeekCloser() - if err != nil { - return err +func (l *genericResource) relTargetPathsForRel(rel string) []string { + if len(l.baseTargetPathDirs) == 0 { + return []string{l.relTargetPathForRelAndBasePath(rel, "", false, false)} } - defer fr.Close() - fw, err := helpers.OpenFilesForWriting(l.spec.BaseFs.PublishFs, l.targetFilenames()...) - if err != nil { - return err + targetPaths := make([]string, len(l.baseTargetPathDirs)) + for i, dir := range l.baseTargetPathDirs { + targetPaths[i] = l.relTargetPathForRelAndBasePath(rel, dir, false, false) } - defer fw.Close() + return targetPaths +} - _, err = io.Copy(fw, fr) - return err +func (l *genericResource) updateParams(params map[string]interface{}) { + if l.params == nil { + l.params = params + return + } + + // Sets the params not already set + for k, v := range params { + if _, found := l.params[k]; !found { + l.params[k] = v + } + } } -// Path is stored with Unix style slashes. -func (l *genericResource) TargetPath() string { - return l.relTargetDirFile.path() +type targetPather interface { + TargetPath() string } -func (l *genericResource) targetFilenames() []string { - paths := l.relTargetPaths() - for i, p := range paths { - paths[i] = filepath.Clean(p) - } - return paths +type permalinker interface { + targetPather + permalinkFor(target string) string + relPermalinkFor(target string) string + relTargetPaths() []string + relTargetPathsFor(target string) []string } -// TODO(bep) clean up below -func (r *Spec) newGenericResource(sourceFs afero.Fs, - targetPathBuilder func() page.TargetPaths, - osFileInfo os.FileInfo, - sourceFilename, - baseFilename string, - mediaType media.Type) *genericResource { - return r.newGenericResourceWithBase( - sourceFs, - false, - nil, - nil, - targetPathBuilder, - osFileInfo, - sourceFilename, - baseFilename, - mediaType, - ) - -} - -func (r *Spec) newGenericResourceWithBase( - sourceFs afero.Fs, - lazyPublish bool, - openReadSeekerCloser resource.OpenReadSeekCloser, - targetPathBaseDirs []string, - targetPathBuilder func() page.TargetPaths, - osFileInfo os.FileInfo, - sourceFilename, - baseFilename string, - mediaType media.Type) *genericResource { - - if osFileInfo != nil && osFileInfo.IsDir() { - panic(fmt.Sprintf("dirs not supported resource types: %v", osFileInfo)) - } +type resourceContent struct { + content string + contentInit sync.Once +} - // This value is used both to construct URLs and file paths, but start - // with a Unix-styled path. - baseFilename = helpers.ToSlashTrimLeading(baseFilename) - fpath, fname := path.Split(baseFilename) +type resourceFileInfo struct { + // Will be set if this resource is backed by something other than a file. + openReadSeekerCloser resource.OpenReadSeekCloser - var resourceType string - if mediaType.MainType == "image" { - resourceType = mediaType.MainType - } else { - resourceType = mediaType.SubType - } + // This may be set to tell us to look in another filesystem for this resource. + // We, by default, use the sourceFs filesystem in the spec below. + sourceFs afero.Fs - pathDescriptor := resourcePathDescriptor{ - baseTargetPathDirs: helpers.UniqueStringsReuse(targetPathBaseDirs), - targetPathBuilder: targetPathBuilder, - relTargetDirFile: dirFile{dir: fpath, file: fname}, + // Absolute filename to the source, including any content folder path. + // Note that this is absolute in relation to the filesystem it is stored in. + // It can be a base path filesystem, and then this filename will not match + // the path to the file on the real filesystem. + sourceFilename string + + fi os.FileInfo + + // A hash of the source content. Is only calculated in caching situations. + h *resourceHash +} + +func (fi *resourceFileInfo) ReadSeekCloser() (hugio.ReadSeekCloser, error) { + if fi.openReadSeekerCloser != nil { + return fi.openReadSeekerCloser() } - var po *publishOnce - if lazyPublish { - po = &publishOnce{logger: r.Logger} + f, err := fi.getSourceFs().Open(fi.getSourceFilename()) + if err != nil { + return nil, err } + return f, nil +} + +func (fi *resourceFileInfo) getSourceFilename() string { + return fi.sourceFilename +} + +func (fi *resourceFileInfo) setSourceFilename(s string) { + // Make sure it's always loaded by sourceFilename. + fi.openReadSeekerCloser = nil + fi.sourceFilename = s +} + +func (fi *resourceFileInfo) getSourceFs() afero.Fs { + return fi.sourceFs +} + +func (fi *resourceFileInfo) setSourceFs(fs afero.Fs) { + fi.sourceFs = fs +} + +func (fi *resourceFileInfo) hash() (string, error) { + var err error + fi.h.init.Do(func() { + var hash string + var f hugio.ReadSeekCloser + f, err = fi.ReadSeekCloser() + if err != nil { + err = errors.Wrap(err, "failed to open source file") + return + } + defer f.Close() + + hash, err = helpers.MD5FromFileFast(f) + if err != nil { + return + } + fi.h.value = hash + }) - return &genericResource{ - openReadSeekerCloser: openReadSeekerCloser, - publishOnce: po, - resourcePathDescriptor: pathDescriptor, - sourceFs: sourceFs, - osFileInfo: osFileInfo, - sourceFilename: sourceFilename, - mediaType: mediaType, - resourceType: resourceType, - spec: r, - params: make(map[string]interface{}), - name: baseFilename, - title: baseFilename, - resourceContent: &resourceContent{}, - resourceHash: &resourceHash{}, + return fi.h.value, err +} + +func (fi *resourceFileInfo) size() int { + if fi.fi == nil { + return 0 } + + return int(fi.fi.Size()) +} + +type resourceHash struct { + value string + init sync.Once +} + +type resourcePathDescriptor struct { + // The relative target directory and filename. + relTargetDirFile dirFile + + // Callback used to construct a target path relative to its owner. + targetPathBuilder func() page.TargetPaths + + // This will normally be the same as above, but this will only apply to publishing + // of resources. It may be mulltiple values when in multihost mode. + baseTargetPathDirs []string + + // baseOffset is set when the output format's path has a offset, e.g. for AMP. + baseOffset string } diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go index 5a5839735..32c76fc83 100644 --- a/resources/resource/resourcetypes.go +++ b/resources/resource/resourcetypes.go @@ -23,7 +23,7 @@ import ( // Cloner is an internal template and not meant for use in the templates. It // may change without notice. type Cloner interface { - WithNewBase(base string) Resource + Clone() Resource } // Resource represents a linkable resource, i.e. a content page, image etc. @@ -35,6 +35,20 @@ type Resource interface { ResourceDataProvider } +// Image represents an image resource. +type Image interface { + Resource + ImageOps +} + +type ImageOps interface { + Height() int + Width() int + Fill(spec string) (Image, error) + Fit(spec string) (Image, error) + Resize(spec string) (Image, error) +} + type ResourceTypesProvider interface { // MediaType is this resource's MIME type. MediaType() media.Type @@ -117,6 +131,10 @@ type OpenReadSeekCloser func() (hugio.ReadSeekCloser, error) // ReadSeekCloserResource is a Resource that supports loading its content. type ReadSeekCloserResource interface { MediaType() media.Type + ReadSeekCloserProvider +} + +type ReadSeekCloserProvider interface { ReadSeekCloser() (hugio.ReadSeekCloser, error) } diff --git a/resources/resource_cache.go b/resources/resource_cache.go index 8f6fcbc0f..47822a7f5 100644 --- a/resources/resource_cache.go +++ b/resources/resource_cache.go @@ -281,7 +281,7 @@ func (c *ResourceCache) DeletePartitions(partitions ...string) { for k := range c.cache { clear := false - for p, _ := range partitionsSet { + for p := range partitionsSet { if strings.Contains(k, p) { // There will be some false positive, but that's fine. clear = true diff --git a/resources/resource_metadata.go b/resources/resource_metadata.go index adb9d6867..79e61e1a0 100644 --- a/resources/resource_metadata.go +++ b/resources/resource_metadata.go @@ -29,9 +29,15 @@ import ( ) var ( - _ metaAssigner = (*genericResource)(nil) + _ metaAssigner = (*genericResource)(nil) + _ metaAssigner = (*imageResource)(nil) + _ metaAssignerProvider = (*resourceAdapter)(nil) ) +type metaAssignerProvider interface { + getMetaAssigner() metaAssigner +} + // metaAssigner allows updating metadata in resources that supports it. type metaAssigner interface { setTitle(title string) @@ -50,8 +56,15 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...resource.Res counters := make(map[string]int) for _, r := range resources { - if _, ok := r.(metaAssigner); !ok { - continue + var ma metaAssigner + mp, ok := r.(metaAssignerProvider) + if ok { + ma = mp.getMetaAssigner() + } else { + ma, ok = r.(metaAssigner) + if !ok { + continue + } } var ( @@ -61,7 +74,6 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...resource.Res resourceSrcKey = strings.ToLower(r.Name()) ) - ma := r.(metaAssigner) for _, meta := range metadata { src, found := meta["src"] if !found { diff --git a/resources/resource_metadata_test.go b/resources/resource_metadata_test.go index bededcd1b..c79a50021 100644 --- a/resources/resource_metadata_test.go +++ b/resources/resource_metadata_test.go @@ -24,7 +24,7 @@ import ( func TestAssignMetadata(t *testing.T) { c := qt.New(t) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) var foo1, foo2, foo3, logo1, logo2, logo3 resource.Resource var resources resource.Resources diff --git a/resources/resource_spec.go b/resources/resource_spec.go new file mode 100644 index 000000000..528a2bd58 --- /dev/null +++ b/resources/resource_spec.go @@ -0,0 +1,304 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "errors" + "fmt" + "mime" + "os" + "path" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/helpers" + + "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/resources/images" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/tpl" + "github.com/spf13/afero" +) + +func NewSpec( + s *helpers.PathSpec, + fileCaches filecache.Caches, + logger *loggers.Logger, + outputFormats output.Formats, + mimeTypes media.Types) (*Spec, error) { + + imgConfig, err := images.DecodeConfig(s.Cfg.GetStringMap("imaging")) + if err != nil { + return nil, err + } + + imaging := &images.ImageProcessor{Cfg: imgConfig} + + if logger == nil { + logger = loggers.NewErrorLogger() + } + + permalinks, err := page.NewPermalinkExpander(s) + if err != nil { + return nil, err + } + + rs := &Spec{PathSpec: s, + Logger: logger, + imaging: imaging, + MediaTypes: mimeTypes, + OutputFormats: outputFormats, + Permalinks: permalinks, + FileCaches: fileCaches, + imageCache: newImageCache( + fileCaches.ImageCache(), + + s, + )} + + rs.ResourceCache = newResourceCache(rs) + + return rs, nil + +} + +type Spec struct { + *helpers.PathSpec + + MediaTypes media.Types + OutputFormats output.Formats + + Logger *loggers.Logger + + TextTemplates tpl.TemplateParseFinder + + Permalinks page.PermalinkExpander + + // Holds default filter settings etc. + imaging *images.ImageProcessor + + imageCache *imageCache + ResourceCache *ResourceCache + FileCaches filecache.Caches +} + +func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { + return r.newResourceFor(fd) +} + +func (r *Spec) CacheStats() string { + r.imageCache.mu.RLock() + defer r.imageCache.mu.RUnlock() + + s := fmt.Sprintf("Cache entries: %d", len(r.imageCache.store)) + + count := 0 + for k := range r.imageCache.store { + if count > 5 { + break + } + s += "\n" + k + count++ + } + + return s +} + +func (r *Spec) ClearCaches() { + r.imageCache.clear() + r.ResourceCache.clear() +} + +func (r *Spec) DeleteCacheByPrefix(prefix string) { + r.imageCache.deleteByPrefix(prefix) +} + +// TODO(bep) unify +func (r *Spec) IsInImageCache(key string) bool { + // This is used for cache pruning. We currently only have images, but we could + // imagine expanding on this. + return r.imageCache.isInCache(key) +} + +func (s *Spec) String() string { + return "spec" +} + +// TODO(bep) clean up below +func (r *Spec) newGenericResource(sourceFs afero.Fs, + targetPathBuilder func() page.TargetPaths, + osFileInfo os.FileInfo, + sourceFilename, + baseFilename string, + mediaType media.Type) *genericResource { + return r.newGenericResourceWithBase( + sourceFs, + nil, + nil, + targetPathBuilder, + osFileInfo, + sourceFilename, + baseFilename, + mediaType, + ) + +} + +func (r *Spec) newGenericResourceWithBase( + sourceFs afero.Fs, + openReadSeekerCloser resource.OpenReadSeekCloser, + targetPathBaseDirs []string, + targetPathBuilder func() page.TargetPaths, + osFileInfo os.FileInfo, + sourceFilename, + baseFilename string, + mediaType media.Type) *genericResource { + + if osFileInfo != nil && osFileInfo.IsDir() { + panic(fmt.Sprintf("dirs not supported resource types: %v", osFileInfo)) + } + + // This value is used both to construct URLs and file paths, but start + // with a Unix-styled path. + baseFilename = helpers.ToSlashTrimLeading(baseFilename) + fpath, fname := path.Split(baseFilename) + + var resourceType string + if mediaType.MainType == "image" { + resourceType = mediaType.MainType + } else { + resourceType = mediaType.SubType + } + + pathDescriptor := &resourcePathDescriptor{ + baseTargetPathDirs: helpers.UniqueStringsReuse(targetPathBaseDirs), + targetPathBuilder: targetPathBuilder, + relTargetDirFile: dirFile{dir: fpath, file: fname}, + } + + gfi := &resourceFileInfo{ + fi: osFileInfo, + openReadSeekerCloser: openReadSeekerCloser, + sourceFs: sourceFs, + sourceFilename: sourceFilename, + h: &resourceHash{}, + } + + g := &genericResource{ + resourceFileInfo: gfi, + resourcePathDescriptor: pathDescriptor, + mediaType: mediaType, + resourceType: resourceType, + spec: r, + params: make(map[string]interface{}), + name: baseFilename, + title: baseFilename, + resourceContent: &resourceContent{}, + } + + return g + +} + +func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) { + fi := fd.FileInfo + var sourceFilename string + + if fd.OpenReadSeekCloser != nil { + } else if fd.SourceFilename != "" { + var err error + fi, err = sourceFs.Stat(fd.SourceFilename) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + sourceFilename = fd.SourceFilename + } else { + sourceFilename = fd.SourceFile.Filename() + } + + if fd.RelTargetFilename == "" { + fd.RelTargetFilename = sourceFilename + } + + ext := strings.ToLower(filepath.Ext(fd.RelTargetFilename)) + mimeType, found := r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, ".")) + // TODO(bep) we need to handle these ambigous types better, but in this context + // we most likely want the application/xml type. + if mimeType.Suffix() == "xml" && mimeType.SubType == "rss" { + mimeType, found = r.MediaTypes.GetByType("application/xml") + } + + if !found { + // A fallback. Note that mime.TypeByExtension is slow by Hugo standards, + // so we should configure media types to avoid this lookup for most + // situations. + mimeStr := mime.TypeByExtension(ext) + if mimeStr != "" { + mimeType, _ = media.FromStringAndExt(mimeStr, ext) + } + } + + gr := r.newGenericResourceWithBase( + sourceFs, + fd.OpenReadSeekCloser, + fd.TargetBasePaths, + fd.TargetPaths, + fi, + sourceFilename, + fd.RelTargetFilename, + mimeType) + + if mimeType.MainType == "image" { + imgFormat, ok := images.ImageFormatFromExt(ext) + if ok { + ir := &imageResource{ + Image: images.NewImage(imgFormat, r.imaging, nil, gr), + baseResource: gr, + } + return newResourceAdapter(gr.spec, fd.LazyPublish, ir), nil + } + + } + + return newResourceAdapter(gr.spec, fd.LazyPublish, gr), nil + +} + +func (r *Spec) newResourceFor(fd ResourceSourceDescriptor) (resource.Resource, error) { + if fd.OpenReadSeekCloser == nil { + if fd.SourceFile != nil && fd.SourceFilename != "" { + return nil, errors.New("both SourceFile and AbsSourceFilename provided") + } else if fd.SourceFile == nil && fd.SourceFilename == "" { + return nil, errors.New("either SourceFile or AbsSourceFilename must be provided") + } + } + + if fd.RelTargetFilename == "" { + fd.RelTargetFilename = fd.Filename() + } + + if len(fd.TargetBasePaths) == 0 { + // If not set, we publish the same resource to all hosts. + fd.TargetBasePaths = r.MultihostTargetBasePaths + } + + return r.newResource(fd.Fs, fd) +} diff --git a/resources/resource_test.go b/resources/resource_test.go index b6d93c9a6..46391527d 100644 --- a/resources/resource_test.go +++ b/resources/resource_test.go @@ -32,7 +32,7 @@ import ( func TestGenericResource(t *testing.T) { c := qt.New(t) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) r := spec.newGenericResource(nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType) @@ -44,7 +44,7 @@ func TestGenericResource(t *testing.T) { func TestGenericResourceWithLinkFacory(t *testing.T) { c := qt.New(t) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) factory := newTargetPaths("/foo") @@ -58,7 +58,7 @@ func TestGenericResourceWithLinkFacory(t *testing.T) { func TestNewResourceFromFilename(t *testing.T) { c := qt.New(t) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) writeSource(t, spec.Fs, "content/a/b/logo.png", "image") writeSource(t, spec.Fs, "content/a/b/data.json", "json") @@ -79,14 +79,11 @@ func TestNewResourceFromFilename(t *testing.T) { c.Assert(r, qt.Not(qt.IsNil)) c.Assert(r.ResourceType(), qt.Equals, "json") - cloned := r.(resource.Cloner).WithNewBase("aceof") - c.Assert(cloned.ResourceType(), qt.Equals, r.ResourceType()) - c.Assert(cloned.RelPermalink(), qt.Equals, "/aceof/a/b/data.json") } func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) { c := qt.New(t) - spec := newTestResourceSpecForBaseURL(c, "https://example.com/docs") + spec := newTestResourceSpec(specDescriptor{c: c, baseURL: "https://example.com/docs"}) writeSource(t, spec.Fs, "content/a/b/logo.png", "image") bfs := afero.NewBasePathFs(spec.Fs.Source, "content") @@ -99,8 +96,6 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) { c.Assert(r.ResourceType(), qt.Equals, "image") c.Assert(r.RelPermalink(), qt.Equals, "/docs/a/b/logo.png") c.Assert(r.Permalink(), qt.Equals, "https://example.com/docs/a/b/logo.png") - img := r.(*Image) - c.Assert(img.targetFilenames()[0], qt.Equals, filepath.FromSlash("/a/b/logo.png")) } @@ -108,7 +103,7 @@ var pngType, _ = media.FromStringAndExt("image/png", "png") func TestResourcesByType(t *testing.T) { c := qt.New(t) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), spec.newGenericResource(nil, nil, nil, "/a/logo.png", "logo.css", pngType), @@ -122,7 +117,7 @@ func TestResourcesByType(t *testing.T) { func TestResourcesGetByPrefix(t *testing.T) { c := qt.New(t) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), @@ -151,7 +146,7 @@ func TestResourcesGetByPrefix(t *testing.T) { func TestResourcesGetMatch(t *testing.T) { c := qt.New(t) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) resources := resource.Resources{ spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), @@ -213,7 +208,7 @@ func BenchmarkResourcesMatch(b *testing.B) { // my own curiosity. func BenchmarkResourcesMatchA100(b *testing.B) { c := qt.New(b) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) a100 := strings.Repeat("a", 100) pattern := "a*a*a*a*a*a*a*a*b" @@ -228,7 +223,7 @@ func BenchmarkResourcesMatchA100(b *testing.B) { func benchResources(b *testing.B) resource.Resources { c := qt.New(b) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) var resources resource.Resources for i := 0; i < 30; i++ { @@ -252,7 +247,7 @@ func benchResources(b *testing.B) resource.Resources { func BenchmarkAssignMetadata(b *testing.B) { c := qt.New(b) - spec := newTestResourceSpec(c) + spec := newTestResourceSpec(specDescriptor{c: c}) for i := 0; i < b.N; i++ { b.StopTimer() diff --git a/resources/resource_transformers/htesting/testhelpers.go b/resources/resource_transformers/htesting/testhelpers.go new file mode 100644 index 000000000..eb664ed3a --- /dev/null +++ b/resources/resource_transformers/htesting/testhelpers.go @@ -0,0 +1,80 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package htesting + +import ( + "path/filepath" + + "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/resources" + "github.com/spf13/afero" + "github.com/spf13/viper" +) + +func NewTestResourceSpec() (*resources.Spec, error) { + cfg := viper.New() + cfg.Set("baseURL", "https://example.org") + cfg.Set("publishDir", "public") + + imagingCfg := map[string]interface{}{ + "resampleFilter": "linear", + "quality": 68, + "anchor": "left", + } + + cfg.Set("imaging", imagingCfg) + + fs := hugofs.NewMem(cfg) + + s, err := helpers.NewPathSpec(fs, cfg, nil) + if err != nil { + return nil, err + } + + filecaches, err := filecache.NewCaches(s) + if err != nil { + return nil, err + } + + spec, err := resources.NewSpec(s, filecaches, nil, output.DefaultFormats, media.DefaultTypes) + return spec, err +} + +func NewResourceTransformer(filename, content string) (resources.ResourceTransformer, error) { + spec, err := NewTestResourceSpec() + if err != nil { + return nil, err + } + return NewResourceTransformerForSpec(spec, filename, content) +} + +func NewResourceTransformerForSpec(spec *resources.Spec, filename, content string) (resources.ResourceTransformer, error) { + filename = filepath.FromSlash(filename) + + fs := spec.Fs.Source + if err := afero.WriteFile(fs, filename, []byte(content), 0777); err != nil { + return nil, err + } + + r, err := spec.New(resources.ResourceSourceDescriptor{Fs: fs, SourceFilename: filename}) + if err != nil { + return nil, err + } + + return r.(resources.ResourceTransformer), nil +} diff --git a/resources/resource_transformers/integrity/integrity.go b/resources/resource_transformers/integrity/integrity.go index 95065603d..1b74de7eb 100644 --- a/resources/resource_transformers/integrity/integrity.go +++ b/resources/resource_transformers/integrity/integrity.go @@ -23,6 +23,8 @@ import ( "html/template" "io" + "github.com/gohugoio/hugo/resources/internal" + "github.com/pkg/errors" "github.com/gohugoio/hugo/resources" @@ -46,8 +48,8 @@ type fingerprintTransformation struct { algo string } -func (t *fingerprintTransformation) Key() resources.ResourceTransformationKey { - return resources.NewResourceTransformationKey("fingerprint", t.algo) +func (t *fingerprintTransformation) Key() internal.ResourceTransformationKey { + return internal.NewResourceTransformationKey("fingerprint", t.algo) } // Transform creates a MD5 hash of the Resource content and inserts that hash before @@ -59,7 +61,17 @@ func (t *fingerprintTransformation) Transform(ctx *resources.ResourceTransformat return err } - io.Copy(io.MultiWriter(h, ctx.To), ctx.From) + var w io.Writer + if rc, ok := ctx.From.(io.ReadSeeker); ok { + // This transformation does not change the content, so try to + // avoid writing to To if we can. + defer rc.Seek(0, 0) + w = h + } else { + w = io.MultiWriter(h, ctx.To) + } + + io.Copy(w, ctx.From) d, err := digest(h) if err != nil { return err @@ -91,15 +103,12 @@ func newHash(algo string) (hash.Hash, error) { // the base64-encoded Subresource Integrity hash, so you will have to stay away from // md5 if you plan to use both. // See https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity -func (c *Client) Fingerprint(res resource.Resource, algo string) (resource.Resource, error) { +func (c *Client) Fingerprint(res resources.ResourceTransformer, algo string) (resource.Resource, error) { if algo == "" { algo = defaultHashAlgo } - return c.rs.Transform( - res, - &fingerprintTransformation{algo: algo}, - ) + return res.Transform(&fingerprintTransformation{algo: algo}) } func integrity(algo string, sum []byte) template.HTMLAttr { diff --git a/resources/resource_transformers/integrity/integrity_test.go b/resources/resource_transformers/integrity/integrity_test.go index cb1caa006..3759e6313 100644 --- a/resources/resource_transformers/integrity/integrity_test.go +++ b/resources/resource_transformers/integrity/integrity_test.go @@ -14,9 +14,13 @@ package integrity import ( + "html/template" "testing" + "github.com/gohugoio/hugo/resources/resource" + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/resources/resource_transformers/htesting" ) func TestHashFromAlgo(t *testing.T) { @@ -46,3 +50,23 @@ func TestHashFromAlgo(t *testing.T) { }) } } + +func TestTransform(t *testing.T) { + c := qt.New(t) + + spec, err := htesting.NewTestResourceSpec() + c.Assert(err, qt.IsNil) + client := New(spec) + + r, err := htesting.NewResourceTransformerForSpec(spec, "hugo.txt", "Hugo Rocks!") + c.Assert(err, qt.IsNil) + + transformed, err := client.Fingerprint(r, "") + + c.Assert(err, qt.IsNil) + c.Assert(transformed.RelPermalink(), qt.Equals, "/hugo.a5ad1c6961214a55de53c1ce6e60d27b6b761f54851fa65e33066460dfa6a0db.txt") + c.Assert(transformed.Data(), qt.DeepEquals, map[string]interface{}{"Integrity": template.HTMLAttr("sha256-pa0caWEhSlXeU8HObmDSe2t2H1SFH6ZeMwZkYN+moNs=")}) + content, err := transformed.(resource.ContentProvider).Content() + c.Assert(err, qt.IsNil) + c.Assert(content, qt.Equals, "Hugo Rocks!") +} diff --git a/resources/resource_transformers/minifier/minify.go b/resources/resource_transformers/minifier/minify.go index 952c6a99c..38e3fc93a 100644 --- a/resources/resource_transformers/minifier/minify.go +++ b/resources/resource_transformers/minifier/minify.go @@ -16,6 +16,7 @@ package minifier import ( "github.com/gohugoio/hugo/minifiers" "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/internal" "github.com/gohugoio/hugo/resources/resource" ) @@ -37,8 +38,8 @@ type minifyTransformation struct { m minifiers.Client } -func (t *minifyTransformation) Key() resources.ResourceTransformationKey { - return resources.NewResourceTransformationKey("minify") +func (t *minifyTransformation) Key() internal.ResourceTransformationKey { + return internal.NewResourceTransformationKey("minify") } func (t *minifyTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { @@ -49,11 +50,10 @@ func (t *minifyTransformation) Transform(ctx *resources.ResourceTransformationCt return nil } -func (c *Client) Minify(res resource.Resource) (resource.Resource, error) { - return c.rs.Transform( - res, - &minifyTransformation{ - rs: c.rs, - m: c.m}, - ) +func (c *Client) Minify(res resources.ResourceTransformer) (resource.Resource, error) { + return res.Transform(&minifyTransformation{ + rs: c.rs, + m: c.m, + }) + } diff --git a/resources/resource_transformers/minifier/minify_test.go b/resources/resource_transformers/minifier/minify_test.go new file mode 100644 index 000000000..3f8853520 --- /dev/null +++ b/resources/resource_transformers/minifier/minify_test.go @@ -0,0 +1,43 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package minifier + +import ( + "testing" + + "github.com/gohugoio/hugo/resources/resource" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/resources/resource_transformers/htesting" +) + +func TestTransform(t *testing.T) { + c := qt.New(t) + + spec, err := htesting.NewTestResourceSpec() + c.Assert(err, qt.IsNil) + client := New(spec) + + r, err := htesting.NewResourceTransformerForSpec(spec, "hugo.html", "<h1> Hugo Rocks! </h1>") + c.Assert(err, qt.IsNil) + + transformed, err := client.Minify(r) + c.Assert(err, qt.IsNil) + + c.Assert(transformed.RelPermalink(), qt.Equals, "/hugo.min.html") + content, err := transformed.(resource.ContentProvider).Content() + c.Assert(err, qt.IsNil) + c.Assert(content, qt.Equals, "<h1>Hugo Rocks!</h1>") + +} diff --git a/resources/resource_transformers/postcss/postcss.go b/resources/resource_transformers/postcss/postcss.go index 452627e65..f262a5c91 100644 --- a/resources/resource_transformers/postcss/postcss.go +++ b/resources/resource_transformers/postcss/postcss.go @@ -17,6 +17,7 @@ import ( "io" "path/filepath" + "github.com/gohugoio/hugo/resources/internal" "github.com/spf13/cast" "github.com/gohugoio/hugo/hugofs" @@ -98,8 +99,8 @@ type postcssTransformation struct { rs *resources.Spec } -func (t *postcssTransformation) Key() resources.ResourceTransformationKey { - return resources.NewResourceTransformationKey("postcss", t.options) +func (t *postcssTransformation) Key() internal.ResourceTransformationKey { + return internal.NewResourceTransformationKey("postcss", t.options) } // Transform shells out to postcss-cli to do the heavy lifting. @@ -187,9 +188,6 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC } // Process transforms the given Resource with the PostCSS processor. -func (c *Client) Process(res resource.Resource, options Options) (resource.Resource, error) { - return c.rs.Transform( - res, - &postcssTransformation{rs: c.rs, options: options}, - ) +func (c *Client) Process(res resources.ResourceTransformer, options Options) (resource.Resource, error) { + return res.Transform(&postcssTransformation{rs: c.rs, options: options}) } diff --git a/resources/resource_transformers/templates/execute_as_template.go b/resources/resource_transformers/templates/execute_as_template.go index b3ec3cf43..422f1bbe1 100644 --- a/resources/resource_transformers/templates/execute_as_template.go +++ b/resources/resource_transformers/templates/execute_as_template.go @@ -17,6 +17,7 @@ package templates import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/internal" "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/tpl" "github.com/pkg/errors" @@ -47,8 +48,8 @@ type executeAsTemplateTransform struct { data interface{} } -func (t *executeAsTemplateTransform) Key() resources.ResourceTransformationKey { - return resources.NewResourceTransformationKey("execute-as-template", t.targetPath) +func (t *executeAsTemplateTransform) Key() internal.ResourceTransformationKey { + return internal.NewResourceTransformationKey("execute-as-template", t.targetPath) } func (t *executeAsTemplateTransform) Transform(ctx *resources.ResourceTransformationCtx) error { @@ -63,14 +64,11 @@ func (t *executeAsTemplateTransform) Transform(ctx *resources.ResourceTransforma return templ.Execute(ctx.To, t.data) } -func (c *Client) ExecuteAsTemplate(res resource.Resource, targetPath string, data interface{}) (resource.Resource, error) { - return c.rs.Transform( - res, - &executeAsTemplateTransform{ - rs: c.rs, - targetPath: helpers.ToSlashTrimLeading(targetPath), - textTemplate: c.textTemplate, - data: data, - }, - ) +func (c *Client) ExecuteAsTemplate(res resources.ResourceTransformer, targetPath string, data interface{}) (resource.Resource, error) { + return res.Transform(&executeAsTemplateTransform{ + rs: c.rs, + targetPath: helpers.ToSlashTrimLeading(targetPath), + textTemplate: c.textTemplate, + data: data, + }) } diff --git a/resources/resource_transformers/tocss/scss/client.go b/resources/resource_transformers/tocss/scss/client.go index e69af2f74..ddf51f7fe 100644 --- a/resources/resource_transformers/tocss/scss/client.go +++ b/resources/resource_transformers/tocss/scss/client.go @@ -18,6 +18,7 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugolib/filesystems" "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/internal" "github.com/gohugoio/hugo/resources/resource" "github.com/spf13/afero" @@ -68,7 +69,7 @@ type options struct { to scss.Options } -func (c *Client) ToCSS(res resource.Resource, opts Options) (resource.Resource, error) { +func (c *Client) ToCSS(res resources.ResourceTransformer, opts Options) (resource.Resource, error) { internalOptions := options{ from: opts, } @@ -83,10 +84,7 @@ func (c *Client) ToCSS(res resource.Resource, opts Options) (resource.Resource, internalOptions.to.Precision = 8 } - return c.rs.Transform( - res, - &toCSSTransformation{c: c, options: internalOptions}, - ) + return res.Transform(&toCSSTransformation{c: c, options: internalOptions}) } type toCSSTransformation struct { @@ -94,8 +92,8 @@ type toCSSTransformation struct { options options } -func (t *toCSSTransformation) Key() resources.ResourceTransformationKey { - return resources.NewResourceTransformationKey("tocss", t.options.from) +func (t *toCSSTransformation) Key() internal.ResourceTransformationKey { + return internal.NewResourceTransformationKey("tocss", t.options.from) } func DecodeOptions(m map[string]interface{}) (opts Options, err error) { diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go index 55664535c..adf752ecc 100644 --- a/resources/testhelpers_test.go +++ b/resources/testhelpers_test.go @@ -4,8 +4,6 @@ import ( "path/filepath" "testing" - "github.com/gohugoio/hugo/htesting/hqt" - "image" "io" "io/ioutil" @@ -28,8 +26,10 @@ import ( "github.com/spf13/viper" ) -func newTestResourceSpec(c *qt.C) *Spec { - return newTestResourceSpecForBaseURL(c, "https://example.com/") +type specDescriptor struct { + baseURL string + c *qt.C + fs afero.Fs } func createTestCfg() *viper.Viper { @@ -54,7 +54,20 @@ func createTestCfg() *viper.Viper { } -func newTestResourceSpecForBaseURL(c *qt.C, baseURL string) *Spec { +func newTestResourceSpec(desc specDescriptor) *Spec { + + baseURL := desc.baseURL + if baseURL == "" { + baseURL = "https://example.com/" + } + + afs := desc.fs + if afs == nil { + afs = afero.NewMemMapFs() + } + + c := desc.c + cfg := createTestCfg() cfg.Set("baseURL", baseURL) @@ -66,7 +79,8 @@ func newTestResourceSpecForBaseURL(c *qt.C, baseURL string) *Spec { cfg.Set("imaging", imagingCfg) - fs := hugofs.NewMem(cfg) + fs := hugofs.NewFrom(afs, cfg) + fs.Destination = hugofs.NewCreateCountingFs(fs.Destination) s, err := helpers.NewPathSpec(fs, cfg, nil) c.Assert(err, qt.IsNil) @@ -117,19 +131,23 @@ func newTestResourceOsFs(c *qt.C) *Spec { } -func fetchSunset(c *qt.C) *Image { +func fetchSunset(c *qt.C) resource.Image { return fetchImage(c, "sunset.jpg") } -func fetchImage(c *qt.C, name string) *Image { - spec := newTestResourceSpec(c) +func fetchImage(c *qt.C, name string) resource.Image { + spec := newTestResourceSpec(specDescriptor{c: c}) return fetchImageForSpec(spec, c, name) } - -func fetchImageForSpec(spec *Spec, c *qt.C, name string) *Image { +func fetchImageForSpec(spec *Spec, c *qt.C, name string) resource.Image { r := fetchResourceForSpec(spec, c, name) - c.Assert(r, hqt.IsSameType, &Image{}) - return r.(*Image) + + img := r.(resource.Image) + + c.Assert(img, qt.Not(qt.IsNil)) + c.Assert(img.(specProvider).getSpec(), qt.Not(qt.IsNil)) + + return img } func fetchResourceForSpec(spec *Spec, c *qt.C, name string) resource.ContentResource { diff --git a/resources/transform.go b/resources/transform.go index 379452bb7..72b9479df 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -15,45 +15,63 @@ package resources import ( "bytes" + "fmt" + "io" "path" - "strconv" "strings" + "sync" + + "github.com/spf13/afero" + + bp "github.com/gohugoio/hugo/bufferpool" - "github.com/pkg/errors" + "github.com/gohugoio/hugo/resources/internal" - "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/resources/resource" - "github.com/mitchellh/hashstructure" - - "fmt" - "io" - "sync" "github.com/gohugoio/hugo/media" - - bp "github.com/gohugoio/hugo/bufferpool" ) var ( - _ resource.ContentResource = (*transformedResource)(nil) - _ resource.ReadSeekCloserResource = (*transformedResource)(nil) - _ collections.Slicer = (*transformedResource)(nil) - _ resource.Identifier = (*transformedResource)(nil) + _ resource.ContentResource = (*resourceAdapter)(nil) + _ resource.ReadSeekCloserResource = (*resourceAdapter)(nil) + _ resource.Resource = (*resourceAdapter)(nil) + _ resource.Source = (*resourceAdapter)(nil) + _ resource.Identifier = (*resourceAdapter)(nil) + _ resource.ResourceMetaProvider = (*resourceAdapter)(nil) ) -func (s *Spec) Transform(r resource.Resource, t ResourceTransformation) (resource.Resource, error) { - if r == nil { - return nil, errors.New("got nil Resource in transformation. Make sure you check with 'with' or 'if' when you get a resource, e.g. with resources.Get.") +// These are transformations that need special support in Hugo that may not +// be available when building the theme/site so we write the transformation +// result to disk and reuse if needed for these, +var transformationsToCacheOnDisk = map[string]bool{ + "postcss": true, + "tocss": true, +} + +func newResourceAdapter(spec *Spec, lazyPublish bool, target transformableResource) *resourceAdapter { + var po *publishOnce + if lazyPublish { + po = &publishOnce{} + } + return &resourceAdapter{ + resourceTransformations: &resourceTransformations{}, + resourceAdapterInner: &resourceAdapterInner{ + spec: spec, + publishOnce: po, + target: target, + }, } +} - return &transformedResource{ - Resource: r, - transformation: t, - transformedResourceMetadata: transformedResourceMetadata{MetaData: make(map[string]interface{})}, - cache: s.ResourceCache}, nil +// ResourceTransformation is the interface that a resource transformation step +// needs to implement. +type ResourceTransformation interface { + Key() internal.ResourceTransformationKey + Transform(ctx *ResourceTransformationCtx) error } type ResourceTransformationCtx struct { @@ -95,20 +113,6 @@ func (ctx *ResourceTransformationCtx) AddOutPathIdentifier(identifier string) { ctx.OutPath = ctx.addPathIdentifier(ctx.InPath, identifier) } -func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string { - dir, file := path.Split(inPath) - base, ext := helpers.PathAndExt(file) - return path.Join(dir, (base + identifier + ext)) -} - -// ReplaceOutPathExtension transforming InPath to OutPath replacing the file -// extension, e.g. ".scss" -func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) { - dir, file := path.Split(ctx.InPath) - base, _ := helpers.PathAndExt(file) - ctx.OutPath = path.Join(dir, (base + newExt)) -} - // PublishSourceMap writes the content to the target folder of the main resource // with the ".map" extension added. func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error { @@ -122,240 +126,198 @@ func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error { return err } -// ResourceTransformationKey are provided by the different transformation implementations. -// It identifies the transformation (name) and its configuration (elements). -// We combine this in a chain with the rest of the transformations -// with the target filename and a content hash of the origin to use as cache key. -type ResourceTransformationKey struct { - name string - elements []interface{} +// ReplaceOutPathExtension transforming InPath to OutPath replacing the file +// extension, e.g. ".scss" +func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) { + dir, file := path.Split(ctx.InPath) + base, _ := helpers.PathAndExt(file) + ctx.OutPath = path.Join(dir, (base + newExt)) } -// NewResourceTransformationKey creates a new ResourceTransformationKey from the transformation -// name and elements. We will create a 64 bit FNV hash from the elements, which when combined -// with the other key elements should be unique for all practical applications. -func NewResourceTransformationKey(name string, elements ...interface{}) ResourceTransformationKey { - return ResourceTransformationKey{name: name, elements: elements} +func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string { + dir, file := path.Split(inPath) + base, ext := helpers.PathAndExt(file) + return path.Join(dir, (base + identifier + ext)) } -// Do not change this without good reasons. -func (k ResourceTransformationKey) key() string { - if len(k.elements) == 0 { - return k.name - } +type publishOnce struct { + publisherInit sync.Once + publisherErr error +} - sb := bp.GetBuffer() - defer bp.PutBuffer(sb) +type resourceAdapter struct { + commonResource + *resourceTransformations + *resourceAdapterInner +} - sb.WriteString(k.name) - for _, element := range k.elements { - hash, err := hashstructure.Hash(element, nil) - if err != nil { - panic(err) - } - sb.WriteString("_") - sb.WriteString(strconv.FormatUint(hash, 10)) +func (r *resourceAdapter) Content() (interface{}, error) { + r.init(false, true) + if r.transformationsErr != nil { + return nil, r.transformationsErr } - - return sb.String() + return r.target.Content() } -// ResourceTransformation is the interface that a resource transformation step -// needs to implement. -type ResourceTransformation interface { - Key() ResourceTransformationKey - Transform(ctx *ResourceTransformationCtx) error +func (r *resourceAdapter) Data() interface{} { + r.init(false, false) + return r.target.Data() } -// We will persist this information to disk. -type transformedResourceMetadata struct { - Target string `json:"Target"` - MediaTypeV string `json:"MediaType"` - MetaData map[string]interface{} `json:"Data"` +func (r *resourceAdapter) Fill(spec string) (resource.Image, error) { + return r.getImageOps().Fill(spec) } -type transformedResource struct { - commonResource +func (r *resourceAdapter) Fit(spec string) (resource.Image, error) { + return r.getImageOps().Fit(spec) +} - cache *ResourceCache +func (r *resourceAdapter) Height() int { + return r.getImageOps().Height() +} - // This is the filename inside resources/_gen/assets - sourceFilename string +func (r *resourceAdapter) Key() string { + r.init(false, false) + return r.target.(resource.Identifier).Key() +} - linker permalinker +func (r *resourceAdapter) MediaType() media.Type { + r.init(false, false) + return r.target.MediaType() +} - // The transformation to apply. - transformation ResourceTransformation +func (r *resourceAdapter) Name() string { + r.init(false, false) + return r.target.Name() +} - // We apply the tranformations lazily. - transformInit sync.Once - transformErr error +func (r *resourceAdapter) Params() map[string]interface{} { + r.init(false, false) + return r.target.Params() +} - // We delay publishing until either .RelPermalink or .Permalink - // is invoked. - publishInit sync.Once - published bool +func (r *resourceAdapter) Permalink() string { + r.init(true, false) + return r.target.Permalink() +} - // The transformed values - content string - contentInit sync.Once - transformedResourceMetadata +func (r *resourceAdapter) Publish() error { + r.init(false, false) - // The source - resource.Resource + return r.target.Publish() } -func (r *transformedResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) { - if err := r.initContent(); err != nil { - return nil, err - } - return hugio.NewReadSeekerNoOpCloserFromString(r.content), nil +func (r *resourceAdapter) ReadSeekCloser() (hugio.ReadSeekCloser, error) { + r.init(false, false) + return r.target.ReadSeekCloser() } -func (r *transformedResource) transferTransformedValues(another *transformedResource) { - if another.content != "" { - r.contentInit.Do(func() { - r.content = another.content - }) - } - r.transformedResourceMetadata = another.transformedResourceMetadata +func (r *resourceAdapter) RelPermalink() string { + r.init(true, false) + return r.target.RelPermalink() } -func (r *transformedResource) tryTransformedFileCache(key string) io.ReadCloser { - fi, f, meta, found := r.cache.getFromFile(key) - if !found { - return nil - } - r.transformedResourceMetadata = meta - r.sourceFilename = fi.Name +func (r *resourceAdapter) Resize(spec string) (resource.Image, error) { + return r.getImageOps().Resize(spec) +} - return f +func (r *resourceAdapter) ResourceType() string { + r.init(false, false) + return r.target.ResourceType() } -func (r *transformedResource) Content() (interface{}, error) { - if err := r.initTransform(true, false); err != nil { - return nil, err - } - if err := r.initContent(); err != nil { - return "", err - } - return r.content, nil +func (r *resourceAdapter) String() string { + return r.Name() } -func (r *transformedResource) Data() interface{} { - if err := r.initTransform(false, false); err != nil { - return noData - } - return r.MetaData +func (r *resourceAdapter) Title() string { + r.init(false, false) + return r.target.Title() } -func (r *transformedResource) MediaType() media.Type { - if err := r.initTransform(false, false); err != nil { - return media.Type{} +func (r resourceAdapter) Transform(t ...ResourceTransformation) (ResourceTransformer, error) { + r.resourceTransformations = &resourceTransformations{ + transformations: append(r.transformations, t...), } - m, _ := r.cache.rs.MediaTypes.GetByType(r.MediaTypeV) - return m -} -func (r *transformedResource) Key() string { - if err := r.initTransform(false, false); err != nil { - return "" + r.resourceAdapterInner = &resourceAdapterInner{ + spec: r.spec, + publishOnce: &publishOnce{}, + target: r.target, } - return r.linker.relPermalinkFor(r.Target) + + return &r, nil } -func (r *transformedResource) Permalink() string { - if err := r.initTransform(false, true); err != nil { - return "" - } - return r.linker.permalinkFor(r.Target) +func (r *resourceAdapter) Width() int { + return r.getImageOps().Width() } -func (r *transformedResource) RelPermalink() string { - if err := r.initTransform(false, true); err != nil { - return "" +func (r *resourceAdapter) getImageOps() resource.ImageOps { + img, ok := r.target.(resource.ImageOps) + if !ok { + panic(fmt.Sprintf("%T is not an image", r.target)) } - return r.linker.relPermalinkFor(r.Target) + r.init(false, false) + return img } -func (r *transformedResource) initContent() error { - var err error - r.contentInit.Do(func() { - var b []byte - _, b, err = r.cache.fileCache.GetBytes(r.sourceFilename) - if err != nil { - return - } - r.content = string(b) - }) - return err +func (r *resourceAdapter) getMetaAssigner() metaAssigner { + return r.target } -func (r *transformedResource) openPublishFileForWriting(relTargetPath string) (io.WriteCloser, error) { - return helpers.OpenFilesForWriting(r.cache.rs.PublishFs, r.linker.relTargetPathsFor(relTargetPath)...) +func (r *resourceAdapter) getSpec() *Spec { + return r.spec } -func (r *transformedResource) transform(setContent, publish bool) (err error) { +func (r *resourceAdapter) publish() { + if r.publishOnce == nil { + return + } - // This can be the last resource in a chain. - // Rewind and create a processing chain. - var chain []resource.Resource - current := r - for { - rr := current.Resource - chain = append(chain[:0], append([]resource.Resource{rr}, chain[0:]...)...) - if tr, ok := rr.(*transformedResource); ok { - current = tr - } else { - break + r.publisherInit.Do(func() { + r.publisherErr = r.target.Publish() + + if r.publisherErr != nil { + r.spec.Logger.ERROR.Printf("Failed to publish Resource: %s", r.publisherErr) } - } + }) - // Append the current transformer at the end - chain = append(chain, r) +} - first := chain[0] +func (r *resourceAdapter) transform(publish, setContent bool) error { + cache := r.spec.ResourceCache // Files with a suffix will be stored in cache (both on disk and in memory) - // partitioned by their suffix. There will be other files below /other. - // This partition is also how we determine what to delete on server reloads. - var key, base string - for _, element := range chain { - switch v := element.(type) { - case *transformedResource: - key = key + "_" + v.transformation.Key().key() - case permalinker: - r.linker = v - p := v.TargetPath() - if p == "" { - panic("target path needed for key creation") - } - base = ResourceCacheKey(p) - default: - return fmt.Errorf("transformation not supported for type %T", element) - } + // partitioned by their suffix. + var key string + for _, tr := range r.transformations { + key = key + "_" + tr.Key().Value() } - key = r.cache.cleanKey(base) + "_" + helpers.MD5String(key) + base := ResourceCacheKey(r.target.TargetPath()) + + key = cache.cleanKey(base) + "_" + helpers.MD5String(key) + + cached, found := cache.get(key) - cached, found := r.cache.get(key) if found { - r.transferTransformedValues(cached.(*transformedResource)) - return + r.resourceAdapterInner = cached.(*resourceAdapterInner) + return nil } // Acquire a write lock for the named transformation. - r.cache.nlocker.Lock(key) + cache.nlocker.Lock(key) // Check the cache again. - cached, found = r.cache.get(key) + cached, found = cache.get(key) if found { - r.transferTransformedValues(cached.(*transformedResource)) - r.cache.nlocker.Unlock(key) - return + r.resourceAdapterInner = cached.(*resourceAdapterInner) + cache.nlocker.Unlock(key) + return nil } - defer r.cache.nlocker.Unlock(key) - defer r.cache.set(key, r) + defer cache.nlocker.Unlock(key) + defer cache.set(key, r.resourceAdapterInner) b1 := bp.GetBuffer() b2 := bp.GetBuffer() @@ -363,68 +325,77 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) { defer bp.PutBuffer(b2) tctx := &ResourceTransformationCtx{ - Data: r.transformedResourceMetadata.MetaData, - OpenResourcePublisher: r.openPublishFileForWriting, + Data: make(map[string]interface{}), + OpenResourcePublisher: r.target.openPublishFileForWriting, } - tctx.InMediaType = first.MediaType() - tctx.OutMediaType = first.MediaType() + tctx.InMediaType = r.target.MediaType() + tctx.OutMediaType = r.target.MediaType() + + startCtx := *tctx + updates := &transformationUpdate{startCtx: startCtx} + + var contentrc hugio.ReadSeekCloser - contentrc, err := contentReadSeekerCloser(first) + contentrc, err := contentReadSeekerCloser(r.target) if err != nil { return err } + defer contentrc.Close() tctx.From = contentrc tctx.To = b1 - if r.linker != nil { - tctx.InPath = r.linker.TargetPath() - tctx.SourcePath = tctx.InPath - } + tctx.InPath = r.target.TargetPath() + tctx.SourcePath = tctx.InPath counter := 0 + writeToFileCache := false var transformedContentr io.Reader - for _, element := range chain { - tr, ok := element.(*transformedResource) - if !ok { - continue - } - counter++ - if counter != 1 { + for i, tr := range r.transformations { + if i != 0 { tctx.InMediaType = tctx.OutMediaType } - if counter%2 == 0 { - tctx.From = b1 - b2.Reset() - tctx.To = b2 - } else { - if counter != 1 { - // The first reader is the file. - tctx.From = b2 - } - b1.Reset() - tctx.To = b1 + + if !writeToFileCache { + writeToFileCache = transformationsToCacheOnDisk[tr.Key().Name] } - if err := tr.transformation.Transform(tctx); err != nil { + if i > 0 { + hasWrites := tctx.To.(*bytes.Buffer).Len() > 0 + if hasWrites { + counter++ + // Switch the buffers + if counter%2 == 0 { + tctx.From = b2 + b1.Reset() + tctx.To = b1 + } else { + tctx.From = b1 + b2.Reset() + tctx.To = b2 + } + } + } - if err == herrors.ErrFeatureNotAvailable { + if err = tr.Transform(tctx); err != nil { + if writeToFileCache && err == herrors.ErrFeatureNotAvailable { // This transformation is not available in this // Hugo installation (scss not compiled in, PostCSS not available etc.) // If a prepared bundle for this transformation chain is available, use that. - f := r.tryTransformedFileCache(key) + f := r.target.tryTransformedFileCache(key, updates) if f == nil { errMsg := err.Error() - if tr.transformation.Key().name == "postcss" { + if tr.Key().Name == "postcss" { errMsg = "PostCSS not found; install with \"npm install postcss-cli\". See https://gohugo.io/hugo-pipes/postcss/" } - return fmt.Errorf("%s: failed to transform %q (%s): %s", strings.ToUpper(tr.transformation.Key().name), tctx.InPath, tctx.InMediaType.Type(), errMsg) + return fmt.Errorf("%s: failed to transform %q (%s): %s", strings.ToUpper(tr.Key().Name), tctx.InPath, tctx.InMediaType.Type(), errMsg) } transformedContentr = f + updates.sourceFs = cache.fileCache.Fs defer f.Close() // The reader above is all we need. @@ -442,34 +413,35 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) { } if transformedContentr == nil { - r.Target = tctx.InPath - r.MediaTypeV = tctx.OutMediaType.Type() + updates.updateFromCtx(tctx) } var publishwriters []io.WriteCloser if publish { - publicw, err := r.openPublishFileForWriting(r.Target) + publicw, err := r.target.openPublishFileForWriting(updates.targetPath) if err != nil { - r.transformErr = err return err } - defer publicw.Close() - publishwriters = append(publishwriters, publicw) } if transformedContentr == nil { - // Also write it to the cache - fi, metaw, err := r.cache.writeMeta(key, r.transformedResourceMetadata) - if err != nil { - return err + if writeToFileCache { + // Also write it to the cache + fi, metaw, err := cache.writeMeta(key, updates.toTransformedResourceMetadata()) + if err != nil { + return err + } + updates.sourceFilename = &fi.Name + updates.sourceFs = cache.fileCache.Fs + publishwriters = append(publishwriters, metaw) } - r.sourceFilename = fi.Name - - publishwriters = append(publishwriters, metaw) - if counter > 0 { + // Any transofrmations reading from From must also write to To. + // This means that if the target buffer is empty, we can just reuse + // the original reader. + if b, ok := tctx.To.(*bytes.Buffer); ok && b.Len() > 0 { transformedContentr = tctx.To.(*bytes.Buffer) } else { transformedContentr = contentrc @@ -479,6 +451,8 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) { // Also write it to memory var contentmemw *bytes.Buffer + setContent = setContent || !writeToFileCache + if setContent { contentmemw = bp.GetBuffer() defer bp.PutBuffer(contentmemw) @@ -486,65 +460,111 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) { } publishw := hugio.NewMultiWriteCloser(publishwriters...) - _, r.transformErr = io.Copy(publishw, transformedContentr) + _, err = io.Copy(publishw, transformedContentr) + if err != nil { + return err + } publishw.Close() if setContent { - r.contentInit.Do(func() { - r.content = contentmemw.String() - }) + s := contentmemw.String() + updates.content = &s + } + + newTarget, err := r.target.cloneWithUpdates(updates) + if err != nil { + return err } + r.target = newTarget return nil } -func (r *transformedResource) initTransform(setContent, publish bool) error { - r.transformInit.Do(func() { - r.published = publish - if err := r.transform(setContent, publish); err != nil { - r.transformErr = err - r.cache.rs.Logger.ERROR.Println("error: failed to transform resource:", err) +func (r *resourceAdapter) init(publish, setContent bool) { + r.initTransform(publish, setContent) +} + +func (r *resourceAdapter) initTransform(publish, setContent bool) { + r.transformationsInit.Do(func() { + if len(r.transformations) == 0 { + // Nothing to do. + return + } + + if publish { + // The transformation will write the content directly to + // the destination. + r.publishOnce = nil } + r.transformationsErr = r.transform(publish, setContent) + if r.transformationsErr != nil { + r.spec.Logger.ERROR.Printf("Transformation failed: %s", r.transformationsErr) + } }) - if !publish { - return r.transformErr + if publish && r.publishOnce != nil { + r.publish() } +} - r.publishInit.Do(func() { - if r.published { - return - } +type resourceAdapterInner struct { + target transformableResource - r.published = true + spec *Spec - // Copy the file from cache to /public - _, src, err := r.cache.fileCache.Get(r.sourceFilename) - if src == nil { - panic(fmt.Sprintf("[BUG] resource cache file not found: %q", r.sourceFilename)) - } + // Handles publishing (to /public) if needed. + *publishOnce +} - if err == nil { - defer src.Close() +type resourceTransformations struct { + transformationsInit sync.Once + transformationsErr error + transformations []ResourceTransformation +} - var dst io.WriteCloser - dst, err = r.openPublishFileForWriting(r.Target) - if err == nil { - defer dst.Close() - io.Copy(dst, src) - } - } +type transformableResource interface { + baseResourceInternal - if err != nil { - r.transformErr = err - r.cache.rs.Logger.ERROR.Println("error: failed to publish resource:", err) - return - } + resource.ContentProvider + resource.Resource +} - }) +type transformationUpdate struct { + content *string + sourceFilename *string + sourceFs afero.Fs + targetPath string + mediaType media.Type + data map[string]interface{} - return r.transformErr + startCtx ResourceTransformationCtx +} + +func (u *transformationUpdate) isContenChanged() bool { + return u.content != nil || u.sourceFilename != nil +} + +func (u *transformationUpdate) toTransformedResourceMetadata() transformedResourceMetadata { + return transformedResourceMetadata{ + MediaTypeV: u.mediaType.Type(), + Target: u.targetPath, + MetaData: u.data, + } +} + +func (u *transformationUpdate) updateFromCtx(ctx *ResourceTransformationCtx) { + u.targetPath = ctx.OutPath + u.mediaType = ctx.OutMediaType + u.data = ctx.Data + u.targetPath = ctx.InPath +} + +// We will persist this information to disk. +type transformedResourceMetadata struct { + Target string `json:"Target"` + MediaTypeV string `json:"MediaType"` + MetaData map[string]interface{} `json:"Data"` } // contentReadSeekerCloser returns a ReadSeekerCloser if possible for a given Resource. diff --git a/resources/transform_test.go b/resources/transform_test.go index 2019bda39..e7235bc8c 100644 --- a/resources/transform_test.go +++ b/resources/transform_test.go @@ -14,23 +14,427 @@ package resources import ( + "encoding/base64" + "fmt" + "io" + "path/filepath" + "strconv" + "strings" + "sync" "testing" + "github.com/gohugoio/hugo/htesting" + + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/hugofs" + + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources/internal" + + "github.com/gohugoio/hugo/helpers" + + "github.com/gohugoio/hugo/resources/resource" + "github.com/spf13/afero" + qt "github.com/frankban/quicktest" ) -type testStruct struct { - Name string - V1 int64 - V2 int32 - V3 int - V4 uint64 -} +const gopher = `iVBORw0KGgoAAAANSUhEUgAAAEsAAAA8CAAAAAALAhhPAAAFfUlEQVRYw62XeWwUVRzHf2+OPbo9d7tsWyiyaZti6eWGAhISoIGKECEKCAiJJkYTiUgTMYSIosYYBBIUIxoSPIINEBDi2VhwkQrVsj1ESgu9doHWdrul7ba73WNm3vOPtsseM9MdwvvrzTs+8/t95ze/33sI5BqiabU6m9En8oNjduLnAEDLUsQXFF8tQ5oxK3vmnNmDSMtrncks9Hhtt/qeWZapHb1ha3UqYSWVl2ZmpWgaXMXGohQAvmeop3bjTRtv6SgaK/Pb9/bFzUrYslbFAmHPp+3WhAYdr+7GN/YnpN46Opv55VDsJkoEpMrY/vO2BIYQ6LLvm0ThY3MzDzzeSJeeWNyTkgnIE5ePKsvKlcg/0T9QMzXalwXMlj54z4c0rh/mzEfr+FgWEz2w6uk8dkzFAgcARAgNp1ZYef8bH2AgvuStbc2/i6CiWGj98y2tw2l4FAXKkQBIf+exyRnteY83LfEwDQAYCoK+P6bxkZm/0966LxcAAILHB56kgD95PPxltuYcMtFTWw/FKkY/6Opf3GGd9ZF+Qp6mzJxzuRSractOmJrH1u8XTvWFHINNkLQLMR+XHXvfPPHw967raE1xxwtA36IMRfkAAG29/7mLuQcb2WOnsJReZGfpiHsSBX81cvMKywYZHhX5hFPtOqPGWZCXnhWGAu6lX91ElKXSalcLXu3UaOXVay57ZSe5f6Gpx7J2MXAsi7EqSp09b/MirKSyJfnfEEgeDjl8FgDAfvewP03zZ+AJ0m9aFRM8eEHBDRKjfcreDXnZdQuAxXpT2NRJ7xl3UkLBhuVGU16gZiGOgZmrSbRdqkILuL/yYoSXHHkl9KXgqNu3PB8oRg0geC5vFmLjad6mUyTKLmF3OtraWDIfACyXqmephaDABawfpi6tqqBZytfQMqOz6S09iWXhktrRaB8Xz4Yi/8gyABDm5NVe6qq/3VzPrcjELWrebVuyY2T7ar4zQyybUCtsQ5Es1FGaZVrRVQwAgHGW2ZCRZshI5bGQi7HesyE972pOSeMM0dSktlzxRdrlqb3Osa6CCS8IJoQQQgBAbTAa5l5epO34rJszibJI8rxLfGzcp1dRosutGeb2VDNgqYrwTiPNsLxXiPi3dz7LiS1WBRBDBOnqEjyy3aQb+/bLiJzz9dIkscVBBLxMfSEac7kO4Fpkngi0ruNBeSOal+u8jgOuqPz12nryMLCniEjtOOOmpt+KEIqsEdocJjYXwrh9OZqWJQyPCTo67LNS/TdxLAv6R5ZNK9npEjbYdT33gRo4o5oTqR34R+OmaSzDBWsAIPhuRcgyoteNi9gF0KzNYWVItPf2TLoXEg+7isNC7uJkgo1iQWOfRSP9NR11RtbZZ3OMG/VhL6jvx+J1m87+RCfJChAtEBQkSBX2PnSiihc/Twh3j0h7qdYQAoRVsRGmq7HU2QRbaxVGa1D6nIOqaIWRjyRZpHMQKWKpZM5feA+lzC4ZFultV8S6T0mzQGhQohi5I8iw+CsqBSxhFMuwyLgSwbghGb0AiIKkSDmGZVmJSiKihsiyOAUs70UkywooYP0bii9GdH4sfr1UNysd3fUyLLMQN+rsmo3grHl9VNJHbbwxoa47Vw5gupIqrZcjPh9R4Nye3nRDk199V+aetmvVtDRE8/+cbgAAgMIWGb3UA0MGLE9SCbWX670TDy1y98c3D27eppUjsZ6fql3jcd5rUe7+ZIlLNQny3Rd+E5Tct3WVhTM5RBCEdiEK0b6B+/ca2gYU393nFj/n1AygRQxPIUA043M42u85+z2SnssKrPl8Mx76NL3E6eXc3be7OD+H4WHbJkKI8AU8irbITQjZ+0hQcPEgId/Fn/pl9crKH02+5o2b9T/eMx7pKoskYgAAAABJRU5ErkJggg==` + +func gopherPNG() io.Reader { return base64.NewDecoder(base64.StdEncoding, strings.NewReader(gopher)) } -func TestResourceTransformationKey(t *testing.T) { - // We really need this key to be portable across OSes. - key := NewResourceTransformationKey("testing", - testStruct{Name: "test", V1: int64(10), V2: int32(20), V3: 30, V4: uint64(40)}) +func TestTransform(t *testing.T) { c := qt.New(t) - c.Assert("testing_518996646957295636", qt.Equals, key.key()) + + createTransformer := func(spec *Spec, filename, content string) Transformer { + filename = filepath.FromSlash(filename) + fs := spec.Fs.Source + afero.WriteFile(fs, filename, []byte(content), 0777) + r, _ := spec.New(ResourceSourceDescriptor{Fs: fs, SourceFilename: filename}) + return r.(Transformer) + } + + createContentReplacer := func(name, old, new string) ResourceTransformation { + return &testTransformation{ + name: name, + transform: func(ctx *ResourceTransformationCtx) error { + in := helpers.ReaderToString(ctx.From) + in = strings.Replace(in, old, new, 1) + ctx.AddOutPathIdentifier("." + name) + fmt.Fprint(ctx.To, in) + return nil + }, + } + } + + // Verify that we publish the same file once only. + assertNoDuplicateWrites := func(c *qt.C, spec *Spec) { + c.Helper() + d := spec.Fs.Destination.(hugofs.DuplicatesReporter) + c.Assert(d.ReportDuplicates(), qt.Equals, "") + } + + assertShouldExist := func(c *qt.C, spec *Spec, filename string, should bool) { + c.Helper() + exists, _ := helpers.Exists(filepath.FromSlash(filename), spec.Fs.Destination) + c.Assert(exists, qt.Equals, should) + } + + c.Run("All values", func(c *qt.C) { + c.Parallel() + + spec := newTestResourceSpec(specDescriptor{c: c}) + + transformation := &testTransformation{ + name: "test", + transform: func(ctx *ResourceTransformationCtx) error { + // Content + in := helpers.ReaderToString(ctx.From) + in = strings.Replace(in, "blue", "green", 1) + fmt.Fprint(ctx.To, in) + + // Media type + ctx.OutMediaType = media.CSVType + + // Change target + ctx.ReplaceOutPathExtension(".csv") + + // Add some data to context + ctx.Data["mydata"] = "Hugo Rocks!" + + return nil + }, + } + + r := createTransformer(spec, "f1.txt", "color is blue") + + tr, err := r.Transform(transformation) + c.Assert(err, qt.IsNil) + content, err := tr.(resource.ContentProvider).Content() + c.Assert(err, qt.IsNil) + + c.Assert(content, qt.Equals, "color is green") + c.Assert(tr.MediaType(), eq, media.CSVType) + c.Assert(tr.RelPermalink(), qt.Equals, "/f1.csv") + assertShouldExist(c, spec, "public/f1.csv", true) + + data := tr.Data().(map[string]interface{}) + c.Assert(data["mydata"], qt.Equals, "Hugo Rocks!") + + assertNoDuplicateWrites(c, spec) + }) + + c.Run("Meta only", func(c *qt.C) { + c.Parallel() + + spec := newTestResourceSpec(specDescriptor{c: c}) + + transformation := &testTransformation{ + name: "test", + transform: func(ctx *ResourceTransformationCtx) error { + // Change media type only + ctx.OutMediaType = media.CSVType + ctx.ReplaceOutPathExtension(".csv") + + return nil + }, + } + + r := createTransformer(spec, "f1.txt", "color is blue") + + tr, err := r.Transform(transformation) + c.Assert(err, qt.IsNil) + content, err := tr.(resource.ContentProvider).Content() + c.Assert(err, qt.IsNil) + + c.Assert(content, qt.Equals, "color is blue") + c.Assert(tr.MediaType(), eq, media.CSVType) + + // The transformed file should only be published if RelPermalink + // or Permalink is called. + n := htesting.RandIntn(3) + shouldExist := true + switch n { + case 0: + tr.RelPermalink() + case 1: + tr.Permalink() + default: + shouldExist = false + } + + assertShouldExist(c, spec, "public/f1.csv", shouldExist) + assertNoDuplicateWrites(c, spec) + }) + + c.Run("Memory-cached transformation", func(c *qt.C) { + c.Parallel() + + spec := newTestResourceSpec(specDescriptor{c: c}) + + // Two transformations with same id, different behaviour. + t1 := createContentReplacer("t1", "blue", "green") + t2 := createContentReplacer("t1", "color", "car") + + for i, transformation := range []ResourceTransformation{t1, t2} { + r := createTransformer(spec, "f1.txt", "color is blue") + tr, _ := r.Transform(transformation) + content, err := tr.(resource.ContentProvider).Content() + c.Assert(err, qt.IsNil) + c.Assert(content, qt.Equals, "color is green", qt.Commentf("i=%d", i)) + + assertShouldExist(c, spec, "public/f1.t1.txt", false) + } + + assertNoDuplicateWrites(c, spec) + }) + + c.Run("File-cached transformation", func(c *qt.C) { + c.Parallel() + + fs := afero.NewMemMapFs() + + for i := 0; i < 2; i++ { + spec := newTestResourceSpec(specDescriptor{c: c, fs: fs}) + + r := createTransformer(spec, "f1.txt", "color is blue") + + var transformation ResourceTransformation + + if i == 0 { + // There is currently a hardcoded list of transformations that we + // persist to disk (tocss, postcss). + transformation = &testTransformation{ + name: "tocss", + transform: func(ctx *ResourceTransformationCtx) error { + in := helpers.ReaderToString(ctx.From) + in = strings.Replace(in, "blue", "green", 1) + ctx.AddOutPathIdentifier("." + "cached") + ctx.OutMediaType = media.CSVType + ctx.Data = map[string]interface{}{ + "Hugo": "Rocks!", + } + fmt.Fprint(ctx.To, in) + return nil + }, + } + } else { + // Force read from file cache. + transformation = &testTransformation{ + name: "tocss", + transform: func(ctx *ResourceTransformationCtx) error { + return herrors.ErrFeatureNotAvailable + }, + } + } + + msg := qt.Commentf("i=%d", i) + + tr, _ := r.Transform(transformation) + c.Assert(tr.RelPermalink(), qt.Equals, "/f1.cached.txt", msg) + content, err := tr.(resource.ContentProvider).Content() + c.Assert(err, qt.IsNil) + c.Assert(content, qt.Equals, "color is green", msg) + c.Assert(tr.MediaType(), eq, media.CSVType) + c.Assert(tr.Data(), qt.DeepEquals, map[string]interface{}{ + "Hugo": "Rocks!", + }) + + assertNoDuplicateWrites(c, spec) + assertShouldExist(c, spec, "public/f1.cached.txt", true) + + } + }) + + c.Run("Access RelPermalink first", func(c *qt.C) { + c.Parallel() + + spec := newTestResourceSpec(specDescriptor{c: c}) + + t1 := createContentReplacer("t1", "blue", "green") + + r := createTransformer(spec, "f1.txt", "color is blue") + + tr, _ := r.Transform(t1) + + relPermalink := tr.RelPermalink() + + content, err := tr.(resource.ContentProvider).Content() + c.Assert(err, qt.IsNil) + + c.Assert(relPermalink, qt.Equals, "/f1.t1.txt") + c.Assert(content, qt.Equals, "color is green") + c.Assert(tr.MediaType(), eq, media.TextType) + + assertNoDuplicateWrites(c, spec) + assertShouldExist(c, spec, "public/f1.t1.txt", true) + }) + + c.Run("Content two", func(c *qt.C) { + c.Parallel() + + spec := newTestResourceSpec(specDescriptor{c: c}) + + t1 := createContentReplacer("t1", "blue", "green") + t2 := createContentReplacer("t1", "color", "car") + + r := createTransformer(spec, "f1.txt", "color is blue") + + tr, _ := r.Transform(t1, t2) + content, err := tr.(resource.ContentProvider).Content() + c.Assert(err, qt.IsNil) + + c.Assert(content, qt.Equals, "car is green") + c.Assert(tr.MediaType(), eq, media.TextType) + + assertNoDuplicateWrites(c, spec) + }) + + c.Run("Content two chained", func(c *qt.C) { + c.Parallel() + + spec := newTestResourceSpec(specDescriptor{c: c}) + + t1 := createContentReplacer("t1", "blue", "green") + t2 := createContentReplacer("t2", "color", "car") + + r := createTransformer(spec, "f1.txt", "color is blue") + + tr1, _ := r.Transform(t1) + tr2, _ := tr1.Transform(t2) + + content1, err := tr1.(resource.ContentProvider).Content() + c.Assert(err, qt.IsNil) + content2, err := tr2.(resource.ContentProvider).Content() + c.Assert(err, qt.IsNil) + + c.Assert(content1, qt.Equals, "color is green") + c.Assert(content2, qt.Equals, "car is green") + + assertNoDuplicateWrites(c, spec) + }) + + c.Run("Content many", func(c *qt.C) { + c.Parallel() + + spec := newTestResourceSpec(specDescriptor{c: c}) + + const count = 26 // A-Z + + transformations := make([]ResourceTransformation, count) + for i := 0; i < count; i++ { + transformations[i] = createContentReplacer(fmt.Sprintf("t%d", i), fmt.Sprint(i), string(i+65)) + } + + var countstr strings.Builder + for i := 0; i < count; i++ { + countstr.WriteString(fmt.Sprint(i)) + } + + r := createTransformer(spec, "f1.txt", countstr.String()) + + tr, _ := r.Transform(transformations...) + content, err := tr.(resource.ContentProvider).Content() + c.Assert(err, qt.IsNil) + + c.Assert(content, qt.Equals, "ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + assertNoDuplicateWrites(c, spec) + }) + + c.Run("Image", func(c *qt.C) { + c.Parallel() + + spec := newTestResourceSpec(specDescriptor{c: c}) + + transformation := &testTransformation{ + name: "test", + transform: func(ctx *ResourceTransformationCtx) error { + ctx.AddOutPathIdentifier(".changed") + return nil + }, + } + + r := createTransformer(spec, "gopher.png", helpers.ReaderToString(gopherPNG())) + + tr, err := r.Transform(transformation) + c.Assert(err, qt.IsNil) + c.Assert(tr.MediaType(), eq, media.PNGType) + + img, ok := tr.(resource.Image) + c.Assert(ok, qt.Equals, true) + + c.Assert(img.Width(), qt.Equals, 75) + c.Assert(img.Height(), qt.Equals, 60) + + // RelPermalink called. + resizedPublished1, err := img.Resize("40x40") + c.Assert(err, qt.IsNil) + c.Assert(resizedPublished1.Height(), qt.Equals, 40) + c.Assert(resizedPublished1.RelPermalink(), qt.Equals, "/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_40x40_resize_linear_2.png") + assertShouldExist(c, spec, "public/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_40x40_resize_linear_2.png", true) + + // Permalink called. + resizedPublished2, err := img.Resize("30x30") + c.Assert(err, qt.IsNil) + c.Assert(resizedPublished2.Height(), qt.Equals, 30) + c.Assert(resizedPublished2.Permalink(), qt.Equals, "https://example.com/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_30x30_resize_linear_2.png") + assertShouldExist(c, spec, "public/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_30x30_resize_linear_2.png", true) + + // Not published because none of RelPermalink or Permalink was called. + resizedNotPublished, err := img.Resize("50x50") + c.Assert(err, qt.IsNil) + c.Assert(resizedNotPublished.Height(), qt.Equals, 50) + //c.Assert(resized.RelPermalink(), qt.Equals, "/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_50x50_resize_linear_2.png") + assertShouldExist(c, spec, "public/gopher.changed_hu2e827f5a78333ebc04166dd643235dea_1462_50x50_resize_linear_2.png", false) + + assertNoDuplicateWrites(c, spec) + + }) + + c.Run("Concurrent", func(c *qt.C) { + spec := newTestResourceSpec(specDescriptor{c: c}) + + transformers := make([]Transformer, 10) + transformations := make([]ResourceTransformation, 10) + + for i := 0; i < 10; i++ { + transformers[i] = createTransformer(spec, fmt.Sprintf("f%d.txt", i), fmt.Sprintf("color is %d", i)) + transformations[i] = createContentReplacer("test", strconv.Itoa(i), "blue") + } + + var wg sync.WaitGroup + + for i := 0; i < 13; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for j := 0; j < 23; j++ { + id := (i + j) % 10 + tr, err := transformers[id].Transform(transformations[id]) + c.Assert(err, qt.IsNil) + content, err := tr.(resource.ContentProvider).Content() + c.Assert(err, qt.IsNil) + c.Assert(content, qt.Equals, "color is blue") + c.Assert(tr.RelPermalink(), qt.Equals, fmt.Sprintf("/f%d.test.txt", id)) + } + }(i) + } + wg.Wait() + + assertNoDuplicateWrites(c, spec) + }) +} + +type testTransformation struct { + name string + transform func(ctx *ResourceTransformationCtx) error +} + +func (t *testTransformation) Key() internal.ResourceTransformationKey { + return internal.NewResourceTransformationKey(t.name) +} + +func (t *testTransformation) Transform(ctx *ResourceTransformationCtx) error { + return t.transform(ctx) } diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go index 3d688e21c..e676a3412 100644 --- a/tpl/resources/resources.go +++ b/tpl/resources/resources.go @@ -22,7 +22,9 @@ import ( _errors "github.com/pkg/errors" "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/resources/resource_factories/bundler" "github.com/gohugoio/hugo/resources/resource_factories/create" "github.com/gohugoio/hugo/resources/resource_transformers/integrity" @@ -174,7 +176,7 @@ func (ns *Namespace) ExecuteAsTemplate(args ...interface{}) (resource.Resource, } data := args[1] - r, ok := args[2].(resource.Resource) + r, ok := args[2].(resources.ResourceTransformer) if !ok { return nil, fmt.Errorf("type %T not supported in Resource transformations", args[2]) } @@ -201,9 +203,9 @@ func (ns *Namespace) Fingerprint(args ...interface{}) (resource.Resource, error) } } - r, ok := args[resIdx].(resource.Resource) + r, ok := args[resIdx].(resources.ResourceTransformer) if !ok { - return nil, fmt.Errorf("%T is not a Resource", args[resIdx]) + return nil, fmt.Errorf("%T can not be transformed", args[resIdx]) } return ns.integrityClient.Fingerprint(r, algo) @@ -211,7 +213,7 @@ func (ns *Namespace) Fingerprint(args ...interface{}) (resource.Resource, error) // Minify minifies the given Resource using the MediaType to pick the correct // minifier. -func (ns *Namespace) Minify(r resource.Resource) (resource.Resource, error) { +func (ns *Namespace) Minify(r resources.ResourceTransformer) (resource.Resource, error) { return ns.minifyClient.Minify(r) } @@ -219,7 +221,7 @@ func (ns *Namespace) Minify(r resource.Resource) (resource.Resource, error) { // object or a target path (string) as first argument. func (ns *Namespace) ToCSS(args ...interface{}) (resource.Resource, error) { var ( - r resource.Resource + r resources.ResourceTransformer m map[string]interface{} targetPath string err error @@ -266,7 +268,7 @@ func (ns *Namespace) PostCSS(args ...interface{}) (resource.Resource, error) { } // We allow string or a map as the first argument in some cases. -func (ns *Namespace) resolveIfFirstArgIsString(args []interface{}) (resource.Resource, string, bool) { +func (ns *Namespace) resolveIfFirstArgIsString(args []interface{}) (resources.ResourceTransformer, string, bool) { if len(args) != 2 { return nil, "", false } @@ -275,26 +277,26 @@ func (ns *Namespace) resolveIfFirstArgIsString(args []interface{}) (resource.Res if !ok1 { return nil, "", false } - v2, ok2 := args[1].(resource.Resource) + v2, ok2 := args[1].(resources.ResourceTransformer) return v2, v1, ok2 } // This roundabout way of doing it is needed to get both pipeline behaviour and options as arguments. -func (ns *Namespace) resolveArgs(args []interface{}) (resource.Resource, map[string]interface{}, error) { +func (ns *Namespace) resolveArgs(args []interface{}) (resources.ResourceTransformer, map[string]interface{}, error) { if len(args) == 0 { return nil, nil, errors.New("no Resource provided in transformation") } if len(args) == 1 { - r, ok := args[0].(resource.Resource) + r, ok := args[0].(resources.ResourceTransformer) if !ok { return nil, nil, fmt.Errorf("type %T not supported in Resource transformations", args[0]) } return r, nil, nil } - r, ok := args[1].(resource.Resource) + r, ok := args[1].(resources.ResourceTransformer) if !ok { return nil, nil, fmt.Errorf("type %T not supported in Resource transformations", args[0]) } |