diff options
author | Bjørn Erik Pedersen <[email protected]> | 2018-08-05 11:13:49 +0200 |
---|---|---|
committer | Bjørn Erik Pedersen <[email protected]> | 2018-08-06 19:58:41 +0200 |
commit | 789ef8c639e4621abd36da530bcb5942ac9297da (patch) | |
tree | f225fc3663affc49805f1d309b77b096d40fc8f6 /transform | |
parent | 71931b30b1813b146aaa60f5cdab16c0f9ebebdb (diff) | |
download | hugo-789ef8c639e4621abd36da530bcb5942ac9297da.tar.gz hugo-789ef8c639e4621abd36da530bcb5942ac9297da.zip |
Add support for minification of final output
Hugo Pipes added minification support for resources fetched via ´resources.Get` and similar.
This also adds support for minification of the final output for supported output formats: HTML, XML, SVG, CSS, JavaScript, JSON.
To enable, run Hugo with the `--minify` flag:
```bash
hugo --minify
```
This commit is also a major spring cleaning of the `transform` package to allow the new minification step fit into that processing chain.
Fixes #1251
Diffstat (limited to 'transform')
-rw-r--r-- | transform/chain.go | 71 | ||||
-rw-r--r-- | transform/chain_test.go | 234 | ||||
-rw-r--r-- | transform/livereloadinject/livereloadinject.go (renamed from transform/livereloadinject.go) | 25 | ||||
-rw-r--r-- | transform/livereloadinject/livereloadinject_test.go (renamed from transform/livereloadinject_test.go) | 10 | ||||
-rw-r--r-- | transform/metainject/hugogenerator.go (renamed from transform/hugogeneratorinject.go) | 26 | ||||
-rw-r--r-- | transform/metainject/hugogenerator_test.go (renamed from transform/hugogeneratorinject_test.go) | 10 | ||||
-rw-r--r-- | transform/urlreplacers/absurl.go (renamed from transform/absurl.go) | 24 | ||||
-rw-r--r-- | transform/urlreplacers/absurlreplacer.go (renamed from transform/absurlreplacer.go) | 22 | ||||
-rw-r--r-- | transform/urlreplacers/absurlreplacer_test.go | 223 |
9 files changed, 352 insertions, 293 deletions
diff --git a/transform/chain.go b/transform/chain.go index f71de94c8..74217dc72 100644 --- a/transform/chain.go +++ b/transform/chain.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,67 +20,74 @@ import ( bp "github.com/gohugoio/hugo/bufferpool" ) -type trans func(rw contentTransformer) +// Transformer is the func that needs to be implemented by a transformation step. +type Transformer func(ft FromTo) error -type link trans +// BytesReader wraps the Bytes method, usually implemented by bytes.Buffer, and an +// io.Reader. +type BytesReader interface { + // The slice given by Bytes is valid for use only until the next buffer modification. + // That is, if you want to use this value outside of the current transformer step, + // you need to take a copy. + Bytes() []byte -type chain []link + io.Reader +} -// NewChain creates a chained content transformer given the provided transforms. -func NewChain(trs ...link) chain { - return trs +// FromTo is sent to each transformation step in the chain. +type FromTo interface { + From() BytesReader + To() io.Writer } -// NewEmptyTransforms creates a new slice of transforms with a capacity of 20. -func NewEmptyTransforms() []link { - return make([]link, 0, 20) +// Chain is an ordered processing chain. The next transform operation will +// receive the output from the previous. +type Chain []Transformer + +// New creates a content transformer chain given the provided transform funcs. +func New(trs ...Transformer) Chain { + return trs } -// contentTransformer is an interface that enables rotation of pooled buffers -// in the transformer chain. -type contentTransformer interface { - Path() []byte - Content() []byte - io.Writer +// NewEmpty creates a new slice of transformers with a capacity of 20. +func NewEmpty() Chain { + return make(Chain, 0, 20) } // Implements contentTransformer // Content is read from the from-buffer and rewritten to to the to-buffer. type fromToBuffer struct { - path []byte from *bytes.Buffer to *bytes.Buffer } -func (ft fromToBuffer) Path() []byte { - return ft.path +func (ft fromToBuffer) From() BytesReader { + return ft.from } -func (ft fromToBuffer) Write(p []byte) (n int, err error) { - return ft.to.Write(p) +func (ft fromToBuffer) To() io.Writer { + return ft.to } -func (ft fromToBuffer) Content() []byte { - return ft.from.Bytes() -} - -func (c *chain) Apply(w io.Writer, r io.Reader, p []byte) error { +// Apply passes the given from io.Reader through the transformation chain. +// The result is written to to. +func (c *Chain) Apply(to io.Writer, from io.Reader) error { if len(*c) == 0 { - _, err := io.Copy(w, r) + _, err := io.Copy(to, from) return err } b1 := bp.GetBuffer() defer bp.PutBuffer(b1) - if _, err := b1.ReadFrom(r); err != nil { + if _, err := b1.ReadFrom(from); err != nil { return err } b2 := bp.GetBuffer() defer bp.PutBuffer(b2) - fb := &fromToBuffer{path: p, from: b1, to: b2} + fb := &fromToBuffer{from: b1, to: b2} for i, tr := range *c { if i > 0 { @@ -95,9 +102,11 @@ func (c *chain) Apply(w io.Writer, r io.Reader, p []byte) error { } } - tr(fb) + if err := tr(fb); err != nil { + return err + } } - _, err := fb.to.WriteTo(w) + _, err := fb.to.WriteTo(to) return err } diff --git a/transform/chain_test.go b/transform/chain_test.go index ae5f06a2d..e34024296 100644 --- a/transform/chain_test.go +++ b/transform/chain_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,142 +15,44 @@ package transform import ( "bytes" - "path/filepath" "strings" "testing" - bp "github.com/gohugoio/hugo/bufferpool" - "github.com/gohugoio/hugo/helpers" "github.com/stretchr/testify/assert" ) -const ( - h5JsContentDoubleQuote = "<!DOCTYPE html><html><head><script src=\"foobar.js\"></script><script src=\"/barfoo.js\"></script></head><body><nav><h1>title</h1></nav><article>content <a href=\"foobar\">foobar</a>. <a href=\"/foobar\">Follow up</a></article></body></html>" - h5JsContentSingleQuote = "<!DOCTYPE html><html><head><script src='foobar.js'></script><script src='/barfoo.js'></script></head><body><nav><h1>title</h1></nav><article>content <a href='foobar'>foobar</a>. <a href='/foobar'>Follow up</a></article></body></html>" - h5JsContentAbsURL = "<!DOCTYPE html><html><head><script src=\"http://user@host:10234/foobar.js\"></script></head><body><nav><h1>title</h1></nav><article>content <a href=\"https://host/foobar\">foobar</a>. Follow up</article></body></html>" - h5JsContentAbsURLSchemaless = "<!DOCTYPE html><html><head><script src=\"//host/foobar.js\"></script><script src='//host2/barfoo.js'></head><body><nav><h1>title</h1></nav><article>content <a href=\"//host/foobar\">foobar</a>. <a href='//host2/foobar'>Follow up</a></article></body></html>" - corectOutputSrcHrefDq = "<!DOCTYPE html><html><head><script src=\"foobar.js\"></script><script src=\"http://base/barfoo.js\"></script></head><body><nav><h1>title</h1></nav><article>content <a href=\"foobar\">foobar</a>. <a href=\"http://base/foobar\">Follow up</a></article></body></html>" - corectOutputSrcHrefSq = "<!DOCTYPE html><html><head><script src='foobar.js'></script><script src='http://base/barfoo.js'></script></head><body><nav><h1>title</h1></nav><article>content <a href='foobar'>foobar</a>. <a href='http://base/foobar'>Follow up</a></article></body></html>" - - h5XMLXontentAbsURL = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?><feed xmlns=\"http://www.w3.org/2005/Atom\"><entry><content type=\"html\"><p><a href="/foobar">foobar</a></p> <p>A video: <iframe src='/foo'></iframe></p></content></entry></feed>" - correctOutputSrcHrefInXML = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?><feed xmlns=\"http://www.w3.org/2005/Atom\"><entry><content type=\"html\"><p><a href="http://base/foobar">foobar</a></p> <p>A video: <iframe src='http://base/foo'></iframe></p></content></entry></feed>" - h5XMLContentGuarded = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?><feed xmlns=\"http://www.w3.org/2005/Atom\"><entry><content type=\"html\"><p><a href="//foobar">foobar</a></p> <p>A video: <iframe src='//foo'></iframe></p></content></entry></feed>" -) - -const ( - // additional sanity tests for replacements testing - replace1 = "No replacements." - replace2 = "ᚠᛇᚻ ᛒᛦᚦ ᚠᚱᚩᚠᚢᚱ\nᚠᛁᚱᚪ ᚷᛖᚻᚹᛦᛚᚳᚢᛗ" - replace3 = `End of file: src="/` - replace4 = `End of file: srcset="/` - replace5 = `Srcsett with no closing quote: srcset="/img/small.jpg do be do be do.` - - // Issue: 816, schemaless links combined with others - replaceSchemalessHTML = `Pre. src='//schemaless' src='/normal' <a href="//schemaless">Schemaless</a>. <a href="/normal">normal</a>. Post.` - replaceSchemalessHTMLCorrect = `Pre. src='//schemaless' src='http://base/normal' <a href="//schemaless">Schemaless</a>. <a href="http://base/normal">normal</a>. Post.` - replaceSchemalessXML = `Pre. src='//schemaless' src='/normal' <a href='//schemaless'>Schemaless</a>. <a href='/normal'>normal</a>. Post.` - replaceSchemalessXMLCorrect = `Pre. src='//schemaless' src='http://base/normal' <a href='//schemaless'>Schemaless</a>. <a href='http://base/normal'>normal</a>. Post.` -) - -const ( - // srcset= - srcsetBasic = `Pre. <img srcset="/img/small.jpg 200w, /img/medium.jpg 300w, /img/big.jpg 700w" alt="text" src="/img/foo.jpg">` - srcsetBasicCorrect = `Pre. <img srcset="http://base/img/small.jpg 200w, http://base/img/medium.jpg 300w, http://base/img/big.jpg 700w" alt="text" src="http://base/img/foo.jpg">` - srcsetSingleQuote = `Pre. <img srcset='/img/small.jpg 200w, /img/big.jpg 700w' alt="text" src="/img/foo.jpg"> POST.` - srcsetSingleQuoteCorrect = `Pre. <img srcset='http://base/img/small.jpg 200w, http://base/img/big.jpg 700w' alt="text" src="http://base/img/foo.jpg"> POST.` - srcsetXMLBasic = `Pre. <img srcset="/img/small.jpg 200w, /img/big.jpg 700w" alt="text" src="/img/foo.jpg">` - srcsetXMLBasicCorrect = `Pre. <img srcset="http://base/img/small.jpg 200w, http://base/img/big.jpg 700w" alt="text" src="http://base/img/foo.jpg">` - srcsetXMLSingleQuote = `Pre. <img srcset="/img/small.jpg 200w, /img/big.jpg 700w" alt="text" src="/img/foo.jpg">` - srcsetXMLSingleQuoteCorrect = `Pre. <img srcset="http://base/img/small.jpg 200w, http://base/img/big.jpg 700w" alt="text" src="http://base/img/foo.jpg">` - srcsetVariations = `Pre. -Missing start quote: <img srcset=/img/small.jpg 200w, /img/big.jpg 700w" alt="text"> src='/img/foo.jpg'> FOO. -<img srcset='/img.jpg'> -schemaless: <img srcset='//img.jpg' src='//basic.jpg'> -schemaless2: <img srcset="//img.jpg" src="//basic.jpg2> POST -` -) - -const ( - srcsetVariationsCorrect = `Pre. -Missing start quote: <img srcset=/img/small.jpg 200w, /img/big.jpg 700w" alt="text"> src='http://base/img/foo.jpg'> FOO. -<img srcset='http://base/img.jpg'> -schemaless: <img srcset='//img.jpg' src='//basic.jpg'> -schemaless2: <img srcset="//img.jpg" src="//basic.jpg2> POST -` - srcsetXMLVariations = `Pre. -Missing start quote: <img srcset=/img/small.jpg 200w /img/big.jpg 700w" alt="text"> src='/img/foo.jpg'> FOO. -<img srcset='/img.jpg'> -schemaless: <img srcset='//img.jpg' src='//basic.jpg'> -schemaless2: <img srcset="//img.jpg" src="//basic.jpg2> POST -` - srcsetXMLVariationsCorrect = `Pre. -Missing start quote: <img srcset=/img/small.jpg 200w /img/big.jpg 700w" alt="text"> src='http://base/img/foo.jpg'> FOO. -<img srcset='http://base/img.jpg'> -schemaless: <img srcset='//img.jpg' src='//basic.jpg'> -schemaless2: <img srcset="//img.jpg" src="//basic.jpg2> POST -` - - relPathVariations = `PRE. a href="/img/small.jpg" POST.` - relPathVariationsCorrect = `PRE. a href="../../img/small.jpg" POST.` - - testBaseURL = "http://base/" -) - -var ( - absURLlBenchTests = []test{ - {h5JsContentDoubleQuote, corectOutputSrcHrefDq}, - {h5JsContentSingleQuote, corectOutputSrcHrefSq}, - {h5JsContentAbsURL, h5JsContentAbsURL}, - {h5JsContentAbsURLSchemaless, h5JsContentAbsURLSchemaless}, - } - - xmlAbsURLBenchTests = []test{ - {h5XMLXontentAbsURL, correctOutputSrcHrefInXML}, - {h5XMLContentGuarded, h5XMLContentGuarded}, - } - - sanityTests = []test{{replace1, replace1}, {replace2, replace2}, {replace3, replace3}, {replace3, replace3}, {replace5, replace5}} - extraTestsHTML = []test{{replaceSchemalessHTML, replaceSchemalessHTMLCorrect}} - absURLTests = append(absURLlBenchTests, append(sanityTests, extraTestsHTML...)...) - extraTestsXML = []test{{replaceSchemalessXML, replaceSchemalessXMLCorrect}} - xmlAbsURLTests = append(xmlAbsURLBenchTests, append(sanityTests, extraTestsXML...)...) - srcsetTests = []test{{srcsetBasic, srcsetBasicCorrect}, {srcsetSingleQuote, srcsetSingleQuoteCorrect}, {srcsetVariations, srcsetVariationsCorrect}} - srcsetXMLTests = []test{ - {srcsetXMLBasic, srcsetXMLBasicCorrect}, - {srcsetXMLSingleQuote, srcsetXMLSingleQuoteCorrect}, - {srcsetXMLVariations, srcsetXMLVariationsCorrect}} - - relurlTests = []test{{relPathVariations, relPathVariationsCorrect}} -) - func TestChainZeroTransformers(t *testing.T) { - tr := NewChain() + tr := New() in := new(bytes.Buffer) out := new(bytes.Buffer) - if err := tr.Apply(in, out, []byte("")); err != nil { + if err := tr.Apply(in, out); err != nil { t.Errorf("A zero transformer chain returned an error.") } } func TestChaingMultipleTransformers(t *testing.T) { - f1 := func(ct contentTransformer) { - ct.Write(bytes.Replace(ct.Content(), []byte("f1"), []byte("f1r"), -1)) + f1 := func(ct FromTo) error { + _, err := ct.To().Write(bytes.Replace(ct.From().Bytes(), []byte("f1"), []byte("f1r"), -1)) + return err } - f2 := func(ct contentTransformer) { - ct.Write(bytes.Replace(ct.Content(), []byte("f2"), []byte("f2r"), -1)) + f2 := func(ct FromTo) error { + _, err := ct.To().Write(bytes.Replace(ct.From().Bytes(), []byte("f2"), []byte("f2r"), -1)) + return err } - f3 := func(ct contentTransformer) { - ct.Write(bytes.Replace(ct.Content(), []byte("f3"), []byte("f3r"), -1)) + f3 := func(ct FromTo) error { + _, err := ct.To().Write(bytes.Replace(ct.From().Bytes(), []byte("f3"), []byte("f3r"), -1)) + return err } - f4 := func(ct contentTransformer) { - ct.Write(bytes.Replace(ct.Content(), []byte("f4"), []byte("f4r"), -1)) + f4 := func(ct FromTo) error { + _, err := ct.To().Write(bytes.Replace(ct.From().Bytes(), []byte("f4"), []byte("f4r"), -1)) + return err } - tr := NewChain(f1, f2, f3, f4) + tr := New(f1, f2, f3, f4) out := new(bytes.Buffer) - if err := tr.Apply(out, strings.NewReader("Test: f4 f3 f1 f2 f1 The End."), []byte("")); err != nil { + if err := tr.Apply(out, strings.NewReader("Test: f4 f3 f1 f2 f1 The End.")); err != nil { t.Errorf("Multi transformer chain returned an error: %s", err) } @@ -161,107 +63,7 @@ func TestChaingMultipleTransformers(t *testing.T) { } } -func BenchmarkAbsURL(b *testing.B) { - tr := NewChain(AbsURL) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - apply(b.Errorf, tr, absURLlBenchTests) - } -} - -func BenchmarkAbsURLSrcset(b *testing.B) { - tr := NewChain(AbsURL) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - apply(b.Errorf, tr, srcsetTests) - } -} - -func BenchmarkXMLAbsURLSrcset(b *testing.B) { - tr := NewChain(AbsURLInXML) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - apply(b.Errorf, tr, srcsetXMLTests) - } -} - -func TestAbsURL(t *testing.T) { - tr := NewChain(AbsURL) - - apply(t.Errorf, tr, absURLTests) - -} - -func TestRelativeURL(t *testing.T) { - tr := NewChain(AbsURL) - - applyWithPath(t.Errorf, tr, relurlTests, helpers.GetDottedRelativePath(filepath.FromSlash("/post/sub/"))) - -} - -func TestAbsURLSrcSet(t *testing.T) { - tr := NewChain(AbsURL) - - apply(t.Errorf, tr, srcsetTests) -} - -func TestAbsXMLURLSrcSet(t *testing.T) { - tr := NewChain(AbsURLInXML) - - apply(t.Errorf, tr, srcsetXMLTests) -} - -func BenchmarkXMLAbsURL(b *testing.B) { - tr := NewChain(AbsURLInXML) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - apply(b.Errorf, tr, xmlAbsURLBenchTests) - } -} - -func TestXMLAbsURL(t *testing.T) { - tr := NewChain(AbsURLInXML) - apply(t.Errorf, tr, xmlAbsURLTests) -} - func TestNewEmptyTransforms(t *testing.T) { - transforms := NewEmptyTransforms() + transforms := NewEmpty() assert.Equal(t, 20, cap(transforms)) } - -type errorf func(string, ...interface{}) - -func applyWithPath(ef errorf, tr chain, tests []test, path string) { - out := bp.GetBuffer() - defer bp.PutBuffer(out) - - in := bp.GetBuffer() - defer bp.PutBuffer(in) - - for _, test := range tests { - var err error - in.WriteString(test.content) - err = tr.Apply(out, in, []byte(path)) - if err != nil { - ef("Unexpected error: %s", err) - } - if test.expected != out.String() { - ef("Expected:\n%s\nGot:\n%s", test.expected, out.String()) - } - out.Reset() - in.Reset() - } -} - -func apply(ef errorf, tr chain, tests []test) { - applyWithPath(ef, tr, tests, testBaseURL) -} - -type test struct { - content string - expected string -} diff --git a/transform/livereloadinject.go b/transform/livereloadinject/livereloadinject.go index 4efd0151d..e04b977f7 100644 --- a/transform/livereloadinject.go +++ b/transform/livereloadinject/livereloadinject.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,30 +11,37 @@ // See the License for the specific language governing permissions and // limitations under the License. -package transform +package livereloadinject import ( "bytes" "fmt" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/transform" ) -// LiveReloadInject returns a function that can be used +// New creates a function that can be used // to inject a script tag for the livereload JavaScript in a HTML document. -func LiveReloadInject(port int) func(ct contentTransformer) { - return func(ct contentTransformer) { +func New(port int) transform.Transformer { + return func(ft transform.FromTo) error { + b := ft.From().Bytes() endBodyTag := "</body>" match := []byte(endBodyTag) replaceTemplate := `<script data-no-instant>document.write('<script src="/livereload.js?port=%d&mindelay=10"></' + 'script>')</script>%s` replace := []byte(fmt.Sprintf(replaceTemplate, port, endBodyTag)) - newcontent := bytes.Replace(ct.Content(), match, replace, 1) - if len(newcontent) == len(ct.Content()) { + newcontent := bytes.Replace(b, match, replace, 1) + if len(newcontent) == len(b) { endBodyTag = "</BODY>" replace := []byte(fmt.Sprintf(replaceTemplate, port, endBodyTag)) match := []byte(endBodyTag) - newcontent = bytes.Replace(ct.Content(), match, replace, 1) + newcontent = bytes.Replace(b, match, replace, 1) } - ct.Write(newcontent) + if _, err := ft.To().Write(newcontent); err != nil { + helpers.DistinctWarnLog.Println("Failed to inject LiveReload script:", err) + } + return nil } } diff --git a/transform/livereloadinject_test.go b/transform/livereloadinject/livereloadinject_test.go index 3337243bd..0e0f708d3 100644 --- a/transform/livereloadinject_test.go +++ b/transform/livereloadinject/livereloadinject_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,13 +11,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package transform +package livereloadinject import ( "bytes" "fmt" "strings" "testing" + + "github.com/gohugoio/hugo/transform" ) func TestLiveReloadInject(t *testing.T) { @@ -29,8 +31,8 @@ func doTestLiveReloadInject(t *testing.T, bodyEndTag string) { out := new(bytes.Buffer) in := strings.NewReader(bodyEndTag) - tr := NewChain(LiveReloadInject(1313)) - tr.Apply(out, in, []byte("path")) + tr := transform.New(New(1313)) + tr.Apply(out, in) expected := fmt.Sprintf(`<script data-no-instant>document.write('<script src="/livereload.js?port=1313&mindelay=10"></' + 'script>')</script>%s`, bodyEndTag) if string(out.Bytes()) != expected { diff --git a/transform/hugogeneratorinject.go b/transform/metainject/hugogenerator.go index 874053087..513b21228 100644 --- a/transform/hugogeneratorinject.go +++ b/transform/metainject/hugogenerator.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package transform +package metainject import ( "bytes" @@ -19,32 +19,36 @@ import ( "regexp" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/transform" ) var metaTagsCheck = regexp.MustCompile(`(?i)<meta\s+name=['|"]?generator['|"]?`) var hugoGeneratorTag = fmt.Sprintf(`<meta name="generator" content="Hugo %s" />`, helpers.CurrentHugoVersion) -// HugoGeneratorInject injects a meta generator tag for Hugo if none present. -func HugoGeneratorInject(ct contentTransformer) { - if metaTagsCheck.Match(ct.Content()) { - if _, err := ct.Write(ct.Content()); err != nil { +// HugoGenerator injects a meta generator tag for Hugo if none present. +func HugoGenerator(ft transform.FromTo) error { + b := ft.From().Bytes() + if metaTagsCheck.Match(b) { + if _, err := ft.To().Write(b); err != nil { helpers.DistinctWarnLog.Println("Failed to inject Hugo generator tag:", err) } - return + return nil } head := "<head>" replace := []byte(fmt.Sprintf("%s\n\t%s", head, hugoGeneratorTag)) - newcontent := bytes.Replace(ct.Content(), []byte(head), replace, 1) + newcontent := bytes.Replace(b, []byte(head), replace, 1) - if len(newcontent) == len(ct.Content()) { + if len(newcontent) == len(b) { head := "<HEAD>" replace := []byte(fmt.Sprintf("%s\n\t%s", head, hugoGeneratorTag)) - newcontent = bytes.Replace(ct.Content(), []byte(head), replace, 1) + newcontent = bytes.Replace(b, []byte(head), replace, 1) } - if _, err := ct.Write(newcontent); err != nil { + if _, err := ft.To().Write(newcontent); err != nil { helpers.DistinctWarnLog.Println("Failed to inject Hugo generator tag:", err) } + return nil + } diff --git a/transform/hugogeneratorinject_test.go b/transform/metainject/hugogenerator_test.go index d37fea24e..ffb4c1425 100644 --- a/transform/hugogeneratorinject_test.go +++ b/transform/metainject/hugogenerator_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,12 +11,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package transform +package metainject import ( "bytes" "strings" "testing" + + "github.com/gohugoio/hugo/transform" ) func TestHugoGeneratorInject(t *testing.T) { @@ -48,8 +50,8 @@ func TestHugoGeneratorInject(t *testing.T) { in := strings.NewReader(this.in) out := new(bytes.Buffer) - tr := NewChain(HugoGeneratorInject) - tr.Apply(out, in, []byte("")) + tr := transform.New(HugoGenerator) + tr.Apply(out, in) if out.String() != this.expect { t.Errorf("[%d] Expected \n%q got \n%q", i, this.expect, out.String()) diff --git a/transform/absurl.go b/transform/urlreplacers/absurl.go index 255ac33b6..029d94da2 100644 --- a/transform/absurl.go +++ b/transform/urlreplacers/absurl.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,18 +11,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -package transform +package urlreplacers + +import "github.com/gohugoio/hugo/transform" var ar = newAbsURLReplacer() -// AbsURL replaces relative URLs with absolute ones +// NewAbsURLTransformer replaces relative URLs with absolute ones // in HTML files, using the baseURL setting. -var AbsURL = func(ct contentTransformer) { - ar.replaceInHTML(ct) +func NewAbsURLTransformer(path string) transform.Transformer { + return func(ft transform.FromTo) error { + ar.replaceInHTML(path, ft) + return nil + } } -// AbsURLInXML replaces relative URLs with absolute ones +// NewAbsURLInXMLTransformer replaces relative URLs with absolute ones // in XML files, using the baseURL setting. -var AbsURLInXML = func(ct contentTransformer) { - ar.replaceInXML(ct) +func NewAbsURLInXMLTransformer(path string) transform.Transformer { + return func(ft transform.FromTo) error { + ar.replaceInXML(path, ft) + return nil + } } diff --git a/transform/absurlreplacer.go b/transform/urlreplacers/absurlreplacer.go index c659a94e8..1de6b0ca7 100644 --- a/transform/absurlreplacer.go +++ b/transform/urlreplacers/absurlreplacer.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,12 +11,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package transform +package urlreplacers import ( "bytes" "io" "unicode/utf8" + + "github.com/gohugoio/hugo/transform" ) type matchState int @@ -260,12 +262,12 @@ func (l *absurllexer) replace() { } } -func doReplace(ct contentTransformer, matchers []absURLMatcher) { +func doReplace(path string, ct transform.FromTo, matchers []absURLMatcher) { lexer := &absurllexer{ - content: ct.Content(), - w: ct, - path: ct.Path(), + content: ct.From().Bytes(), + w: ct.To(), + path: []byte(path), matchers: matchers} lexer.replace() @@ -303,10 +305,10 @@ func newAbsURLReplacer() *absURLReplacer { }} } -func (au *absURLReplacer) replaceInHTML(ct contentTransformer) { - doReplace(ct, au.htmlMatchers) +func (au *absURLReplacer) replaceInHTML(path string, ct transform.FromTo) { + doReplace(path, ct, au.htmlMatchers) } -func (au *absURLReplacer) replaceInXML(ct contentTransformer) { - doReplace(ct, au.xmlMatchers) +func (au *absURLReplacer) replaceInXML(path string, ct transform.FromTo) { + doReplace(path, ct, au.xmlMatchers) } diff --git a/transform/urlreplacers/absurlreplacer_test.go b/transform/urlreplacers/absurlreplacer_test.go new file mode 100644 index 000000000..7a530862b --- /dev/null +++ b/transform/urlreplacers/absurlreplacer_test.go @@ -0,0 +1,223 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package urlreplacers + +import ( + "path/filepath" + "testing" + + bp "github.com/gohugoio/hugo/bufferpool" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/transform" +) + +const ( + h5JsContentDoubleQuote = "<!DOCTYPE html><html><head><script src=\"foobar.js\"></script><script src=\"/barfoo.js\"></script></head><body><nav><h1>title</h1></nav><article>content <a href=\"foobar\">foobar</a>. <a href=\"/foobar\">Follow up</a></article></body></html>" + h5JsContentSingleQuote = "<!DOCTYPE html><html><head><script src='foobar.js'></script><script src='/barfoo.js'></script></head><body><nav><h1>title</h1></nav><article>content <a href='foobar'>foobar</a>. <a href='/foobar'>Follow up</a></article></body></html>" + h5JsContentAbsURL = "<!DOCTYPE html><html><head><script src=\"http://user@host:10234/foobar.js\"></script></head><body><nav><h1>title</h1></nav><article>content <a href=\"https://host/foobar\">foobar</a>. Follow up</article></body></html>" + h5JsContentAbsURLSchemaless = "<!DOCTYPE html><html><head><script src=\"//host/foobar.js\"></script><script src='//host2/barfoo.js'></head><body><nav><h1>title</h1></nav><article>content <a href=\"//host/foobar\">foobar</a>. <a href='//host2/foobar'>Follow up</a></article></body></html>" + corectOutputSrcHrefDq = "<!DOCTYPE html><html><head><script src=\"foobar.js\"></script><script src=\"http://base/barfoo.js\"></script></head><body><nav><h1>title</h1></nav><article>content <a href=\"foobar\">foobar</a>. <a href=\"http://base/foobar\">Follow up</a></article></body></html>" + corectOutputSrcHrefSq = "<!DOCTYPE html><html><head><script src='foobar.js'></script><script src='http://base/barfoo.js'></script></head><body><nav><h1>title</h1></nav><article>content <a href='foobar'>foobar</a>. <a href='http://base/foobar'>Follow up</a></article></body></html>" + + h5XMLXontentAbsURL = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?><feed xmlns=\"http://www.w3.org/2005/Atom\"><entry><content type=\"html\"><p><a href="/foobar">foobar</a></p> <p>A video: <iframe src='/foo'></iframe></p></content></entry></feed>" + correctOutputSrcHrefInXML = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?><feed xmlns=\"http://www.w3.org/2005/Atom\"><entry><content type=\"html\"><p><a href="http://base/foobar">foobar</a></p> <p>A video: <iframe src='http://base/foo'></iframe></p></content></entry></feed>" + h5XMLContentGuarded = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?><feed xmlns=\"http://www.w3.org/2005/Atom\"><entry><content type=\"html\"><p><a href="//foobar">foobar</a></p> <p>A video: <iframe src='//foo'></iframe></p></content></entry></feed>" +) + +const ( + // additional sanity tests for replacements testing + replace1 = "No replacements." + replace2 = "ᚠᛇᚻ ᛒᛦᚦ ᚠᚱᚩᚠᚢᚱ\nᚠᛁᚱᚪ ᚷᛖᚻᚹᛦᛚᚳᚢᛗ" + replace3 = `End of file: src="/` + replace4 = `End of file: srcset="/` + replace5 = `Srcsett with no closing quote: srcset="/img/small.jpg do be do be do.` + + // Issue: 816, schemaless links combined with others + replaceSchemalessHTML = `Pre. src='//schemaless' src='/normal' <a href="//schemaless">Schemaless</a>. <a href="/normal">normal</a>. Post.` + replaceSchemalessHTMLCorrect = `Pre. src='//schemaless' src='http://base/normal' <a href="//schemaless">Schemaless</a>. <a href="http://base/normal">normal</a>. Post.` + replaceSchemalessXML = `Pre. src='//schemaless' src='/normal' <a href='//schemaless'>Schemaless</a>. <a href='/normal'>normal</a>. Post.` + replaceSchemalessXMLCorrect = `Pre. src='//schemaless' src='http://base/normal' <a href='//schemaless'>Schemaless</a>. <a href='http://base/normal'>normal</a>. Post.` +) + +const ( + // srcset= + srcsetBasic = `Pre. <img srcset="/img/small.jpg 200w, /img/medium.jpg 300w, /img/big.jpg 700w" alt="text" src="/img/foo.jpg">` + srcsetBasicCorrect = `Pre. <img srcset="http://base/img/small.jpg 200w, http://base/img/medium.jpg 300w, http://base/img/big.jpg 700w" alt="text" src="http://base/img/foo.jpg">` + srcsetSingleQuote = `Pre. <img srcset='/img/small.jpg 200w, /img/big.jpg 700w' alt="text" src="/img/foo.jpg"> POST.` + srcsetSingleQuoteCorrect = `Pre. <img srcset='http://base/img/small.jpg 200w, http://base/img/big.jpg 700w' alt="text" src="http://base/img/foo.jpg"> POST.` + srcsetXMLBasic = `Pre. <img srcset="/img/small.jpg 200w, /img/big.jpg 700w" alt="text" src="/img/foo.jpg">` + srcsetXMLBasicCorrect = `Pre. <img srcset="http://base/img/small.jpg 200w, http://base/img/big.jpg 700w" alt="text" src="http://base/img/foo.jpg">` + srcsetXMLSingleQuote = `Pre. <img srcset="/img/small.jpg 200w, /img/big.jpg 700w" alt="text" src="/img/foo.jpg">` + srcsetXMLSingleQuoteCorrect = `Pre. <img srcset="http://base/img/small.jpg 200w, http://base/img/big.jpg 700w" alt="text" src="http://base/img/foo.jpg">` + srcsetVariations = `Pre. +Missing start quote: <img srcset=/img/small.jpg 200w, /img/big.jpg 700w" alt="text"> src='/img/foo.jpg'> FOO. +<img srcset='/img.jpg'> +schemaless: <img srcset='//img.jpg' src='//basic.jpg'> +schemaless2: <img srcset="//img.jpg" src="//basic.jpg2> POST +` +) + +const ( + srcsetVariationsCorrect = `Pre. +Missing start quote: <img srcset=/img/small.jpg 200w, /img/big.jpg 700w" alt="text"> src='http://base/img/foo.jpg'> FOO. +<img srcset='http://base/img.jpg'> +schemaless: <img srcset='//img.jpg' src='//basic.jpg'> +schemaless2: <img srcset="//img.jpg" src="//basic.jpg2> POST +` + srcsetXMLVariations = `Pre. +Missing start quote: <img srcset=/img/small.jpg 200w /img/big.jpg 700w" alt="text"> src='/img/foo.jpg'> FOO. +<img srcset='/img.jpg'> +schemaless: <img srcset='//img.jpg' src='//basic.jpg'> +schemaless2: <img srcset="//img.jpg" src="//basic.jpg2> POST +` + srcsetXMLVariationsCorrect = `Pre. +Missing start quote: <img srcset=/img/small.jpg 200w /img/big.jpg 700w" alt="text"> src='http://base/img/foo.jpg'> FOO. +<img srcset='http://base/img.jpg'> +schemaless: <img srcset='//img.jpg' src='//basic.jpg'> +schemaless2: <img srcset="//img.jpg" src="//basic.jpg2> POST +` + + relPathVariations = `PRE. a href="/img/small.jpg" POST.` + relPathVariationsCorrect = `PRE. a href="../../img/small.jpg" POST.` + + testBaseURL = "http://base/" +) + +var ( + absURLlBenchTests = []test{ + {h5JsContentDoubleQuote, corectOutputSrcHrefDq}, + {h5JsContentSingleQuote, corectOutputSrcHrefSq}, + {h5JsContentAbsURL, h5JsContentAbsURL}, + {h5JsContentAbsURLSchemaless, h5JsContentAbsURLSchemaless}, + } + + xmlAbsURLBenchTests = []test{ + {h5XMLXontentAbsURL, correctOutputSrcHrefInXML}, + {h5XMLContentGuarded, h5XMLContentGuarded}, + } + + sanityTests = []test{{replace1, replace1}, {replace2, replace2}, {replace3, replace3}, {replace3, replace3}, {replace5, replace5}} + extraTestsHTML = []test{{replaceSchemalessHTML, replaceSchemalessHTMLCorrect}} + absURLTests = append(absURLlBenchTests, append(sanityTests, extraTestsHTML...)...) + extraTestsXML = []test{{replaceSchemalessXML, replaceSchemalessXMLCorrect}} + xmlAbsURLTests = append(xmlAbsURLBenchTests, append(sanityTests, extraTestsXML...)...) + srcsetTests = []test{{srcsetBasic, srcsetBasicCorrect}, {srcsetSingleQuote, srcsetSingleQuoteCorrect}, {srcsetVariations, srcsetVariationsCorrect}} + srcsetXMLTests = []test{ + {srcsetXMLBasic, srcsetXMLBasicCorrect}, + {srcsetXMLSingleQuote, srcsetXMLSingleQuoteCorrect}, + {srcsetXMLVariations, srcsetXMLVariationsCorrect}} + + relurlTests = []test{{relPathVariations, relPathVariationsCorrect}} +) + +func BenchmarkAbsURL(b *testing.B) { + tr := transform.New(NewAbsURLTransformer(testBaseURL)) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + apply(b.Errorf, tr, absURLlBenchTests) + } +} + +func BenchmarkAbsURLSrcset(b *testing.B) { + tr := transform.New(NewAbsURLTransformer(testBaseURL)) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + apply(b.Errorf, tr, srcsetTests) + } +} + +func BenchmarkXMLAbsURLSrcset(b *testing.B) { + tr := transform.New(NewAbsURLInXMLTransformer(testBaseURL)) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + apply(b.Errorf, tr, srcsetXMLTests) + } +} + +func TestAbsURL(t *testing.T) { + tr := transform.New(NewAbsURLTransformer(testBaseURL)) + + apply(t.Errorf, tr, absURLTests) + +} + +func TestRelativeURL(t *testing.T) { + tr := transform.New(NewAbsURLTransformer(helpers.GetDottedRelativePath(filepath.FromSlash("/post/sub/")))) + + applyWithPath(t.Errorf, tr, relurlTests) + +} + +func TestAbsURLSrcSet(t *testing.T) { + tr := transform.New(NewAbsURLTransformer(testBaseURL)) + + apply(t.Errorf, tr, srcsetTests) +} + +func TestAbsXMLURLSrcSet(t *testing.T) { + tr := transform.New(NewAbsURLInXMLTransformer(testBaseURL)) + + apply(t.Errorf, tr, srcsetXMLTests) +} + +func BenchmarkXMLAbsURL(b *testing.B) { + tr := transform.New(NewAbsURLInXMLTransformer("")) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + apply(b.Errorf, tr, xmlAbsURLBenchTests) + } +} + +func TestXMLAbsURL(t *testing.T) { + tr := transform.New(NewAbsURLInXMLTransformer(testBaseURL)) + apply(t.Errorf, tr, xmlAbsURLTests) +} + +func apply(ef errorf, tr transform.Chain, tests []test) { + applyWithPath(ef, tr, tests) +} + +func applyWithPath(ef errorf, tr transform.Chain, tests []test) { + out := bp.GetBuffer() + defer bp.PutBuffer(out) + + in := bp.GetBuffer() + defer bp.PutBuffer(in) + + for _, test := range tests { + var err error + in.WriteString(test.content) + err = tr.Apply(out, in) + if err != nil { + ef("Unexpected error: %s", err) + } + if test.expected != out.String() { + ef("Expected:\n%s\nGot:\n%s", test.expected, out.String()) + } + out.Reset() + in.Reset() + } +} + +type test struct { + content string + expected string +} + +type errorf func(string, ...interface{}) |