diff options
| author | Bjørn Erik Pedersen <[email protected]> | 2020-05-23 15:32:27 +0200 |
|---|---|---|
| committer | Bjørn Erik Pedersen <[email protected]> | 2020-05-23 22:00:34 +0200 |
| commit | 6c3c6686f5d3c7155e2d455b07ac8ab70f42cb88 (patch) | |
| tree | 666c09383480c74ba69ea7a0442a67029b83a095 | |
| parent | c34bf48560c91c8a2fa106867af7b08a569609b5 (diff) | |
| download | hugo-6c3c6686f5d3c7155e2d455b07ac8ab70f42cb88.tar.gz hugo-6c3c6686f5d3c7155e2d455b07ac8ab70f42cb88.zip | |
Fix Go template script escaping
Fixes #6695
27 files changed, 1092 insertions, 294 deletions
diff --git a/hugolib/template_test.go b/hugolib/template_test.go index 9f04aabdd..29993120d 100644 --- a/hugolib/template_test.go +++ b/hugolib/template_test.go @@ -566,6 +566,24 @@ title: P1 } +func TestTemplateGoIssues(t *testing.T) { + b := newTestSitesBuilder(t) + + b.WithTemplatesAdded( + "index.html", ` +{{ $title := "a & b" }} +<script type="application/ld+json">{"@type":"WebPage","headline":"{{$title}}"}</script> +`, + ) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", ` +<script type="application/ld+json">{"@type":"WebPage","headline":"a \u0026 b"}</script> + +`) +} + func collectIdentities(set map[identity.Identity]bool, provider identity.Provider) { if ids, ok := provider.(identity.IdentitiesProvider); ok { for _, id := range ids.GetIdentities() { diff --git a/scripts/fork_go_templates/main.go b/scripts/fork_go_templates/main.go index d9d056797..04202b254 100644 --- a/scripts/fork_go_templates/main.go +++ b/scripts/fork_go_templates/main.go @@ -17,7 +17,7 @@ import ( func main() { // TODO(bep) git checkout tag - // The current is built with Go version 9341fe073e6f7742c9d61982084874560dac2014 / go1.13.5 + // The current is built with Go version b68fa57c599720d33a2d735782969ce95eabf794 / go1.15dev fmt.Println("Forking ...") defer fmt.Println("Done ...") @@ -55,6 +55,8 @@ var ( textTemplateReplacers = strings.NewReplacer( `"text/template/`, `"github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/`, `"internal/fmtsort"`, `"github.com/gohugoio/hugo/tpl/internal/go_templates/fmtsort"`, + `"internal/testenv"`, `"github.com/gohugoio/hugo/tpl/internal/go_templates/testenv"`, + "TestLinkerGC", "_TestLinkerGC", // Rename types and function that we want to overload. "type state struct", "type stateOld struct", "func (s *state) evalFunction", "func (s *state) evalFunctionOld", @@ -63,6 +65,10 @@ var ( "func isTrue(val reflect.Value) (truth, ok bool) {", "func isTrueOld(val reflect.Value) (truth, ok bool) {", ) + testEnvReplacers = strings.NewReplacer( + `"internal/cfg"`, `"github.com/gohugoio/hugo/tpl/internal/go_templates/cfg"`, + ) + htmlTemplateReplacers = strings.NewReplacer( `. "html/template"`, `. "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate"`, `"html/template"`, `template "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate"`, @@ -116,6 +122,13 @@ var goPackages = []goPackage{ goPackage{srcPkg: "internal/fmtsort", dstPkg: "fmtsort", rewriter: func(name string) { rewrite(name, `"internal/fmtsort" -> "github.com/gohugoio/hugo/tpl/internal/go_templates/fmtsort"`) }}, + goPackage{srcPkg: "internal/testenv", dstPkg: "testenv", + replacer: func(name, content string) string { return testEnvReplacers.Replace(content) }, rewriter: func(name string) { + rewrite(name, `"internal/testenv" -> "github.com/gohugoio/hugo/tpl/internal/go_templates/testenv"`) + }}, + goPackage{srcPkg: "internal/cfg", dstPkg: "cfg", rewriter: func(name string) { + rewrite(name, `"internal/cfg" -> "github.com/gohugoio/hugo/tpl/internal/go_templates/cfg"`) + }}, } var fs = afero.NewOsFs() diff --git a/tpl/internal/go_templates/cfg/cfg.go b/tpl/internal/go_templates/cfg/cfg.go new file mode 100644 index 000000000..bdbe9df3e --- /dev/null +++ b/tpl/internal/go_templates/cfg/cfg.go @@ -0,0 +1,64 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cfg holds configuration shared by the Go command and internal/testenv. +// Definitions that don't need to be exposed outside of cmd/go should be in +// cmd/go/internal/cfg instead of this package. +package cfg + +// KnownEnv is a list of environment variables that affect the operation +// of the Go command. +const KnownEnv = ` + AR + CC + CGO_CFLAGS + CGO_CFLAGS_ALLOW + CGO_CFLAGS_DISALLOW + CGO_CPPFLAGS + CGO_CPPFLAGS_ALLOW + CGO_CPPFLAGS_DISALLOW + CGO_CXXFLAGS + CGO_CXXFLAGS_ALLOW + CGO_CXXFLAGS_DISALLOW + CGO_ENABLED + CGO_FFLAGS + CGO_FFLAGS_ALLOW + CGO_FFLAGS_DISALLOW + CGO_LDFLAGS + CGO_LDFLAGS_ALLOW + CGO_LDFLAGS_DISALLOW + CXX + FC + GCCGO + GO111MODULE + GO386 + GOARCH + GOARM + GOBIN + GOCACHE + GOENV + GOEXE + GOFLAGS + GOGCCFLAGS + GOHOSTARCH + GOHOSTOS + GOINSECURE + GOMIPS + GOMIPS64 + GOMODCACHE + GONOPROXY + GONOSUMDB + GOOS + GOPATH + GOPPC64 + GOPRIVATE + GOPROXY + GOROOT + GOSUMDB + GOTMPDIR + GOTOOLDIR + GOWASM + GO_EXTLINK_ENABLED + PKG_CONFIG +` diff --git a/tpl/internal/go_templates/fmtsort/sort.go b/tpl/internal/go_templates/fmtsort/sort.go index 70a305a3a..b01229bd0 100644 --- a/tpl/internal/go_templates/fmtsort/sort.go +++ b/tpl/internal/go_templates/fmtsort/sort.go @@ -53,12 +53,16 @@ func Sort(mapValue reflect.Value) *SortedMap { if mapValue.Type().Kind() != reflect.Map { return nil } - key := make([]reflect.Value, mapValue.Len()) - value := make([]reflect.Value, len(key)) + // Note: this code is arranged to not panic even in the presence + // of a concurrent map update. The runtime is responsible for + // yelling loudly if that happens. See issue 33275. + n := mapValue.Len() + key := make([]reflect.Value, 0, n) + value := make([]reflect.Value, 0, n) iter := mapValue.MapRange() - for i := 0; iter.Next(); i++ { - key[i] = iter.Key() - value[i] = iter.Value() + for iter.Next() { + key = append(key, iter.Key()) + value = append(value, iter.Value()) } sorted := &SortedMap{ Key: key, diff --git a/tpl/internal/go_templates/fmtsort/sort_test.go b/tpl/internal/go_templates/fmtsort/sort_test.go index 601ec9d25..364c5bf6d 100644 --- a/tpl/internal/go_templates/fmtsort/sort_test.go +++ b/tpl/internal/go_templates/fmtsort/sort_test.go @@ -119,7 +119,7 @@ var sortTests = []sortTest{ "PTR0:0 PTR1:1 PTR2:2", }, { - map[toy]string{toy{7, 2}: "72", toy{7, 1}: "71", toy{3, 4}: "34"}, + map[toy]string{{7, 2}: "72", {7, 1}: "71", {3, 4}: "34"}, "{3 4}:34 {7 1}:71 {7 2}:72", }, { diff --git a/tpl/internal/go_templates/htmltemplate/content_test.go b/tpl/internal/go_templates/htmltemplate/content_test.go index 2a1abfbfb..b5de701d3 100644 --- a/tpl/internal/go_templates/htmltemplate/content_test.go +++ b/tpl/internal/go_templates/htmltemplate/content_test.go @@ -21,7 +21,7 @@ func TestTypedContent(t *testing.T) { htmltemplate.HTML(`Hello, <b>World</b> &tc!`), htmltemplate.HTMLAttr(` dir="ltr"`), htmltemplate.JS(`c && alert("Hello, World!");`), - htmltemplate.JSStr(`Hello, World & O'Reilly\x21`), + htmltemplate.JSStr(`Hello, World & O'Reilly\u0021`), htmltemplate.URL(`greeting=H%69,&addressee=(World)`), htmltemplate.Srcset(`greeting=H%69,&addressee=(World) 2x, https://golang.org/favicon.ico 500.5w`), htmltemplate.URL(`,foo/,`), @@ -73,7 +73,7 @@ func TestTypedContent(t *testing.T) { `Hello, <b>World</b> &tc!`, ` dir="ltr"`, `c && alert("Hello, World!");`, - `Hello, World & O'Reilly\x21`, + `Hello, World & O'Reilly\u0021`, `greeting=H%69,&addressee=(World)`, `greeting=H%69,&addressee=(World) 2x, https://golang.org/favicon.ico 500.5w`, `,foo/,`, @@ -103,7 +103,7 @@ func TestTypedContent(t *testing.T) { `Hello, World &tc!`, ` dir="ltr"`, `c && alert("Hello, World!");`, - `Hello, World & O'Reilly\x21`, + `Hello, World & O'Reilly\u0021`, `greeting=H%69,&addressee=(World)`, `greeting=H%69,&addressee=(World) 2x, https://golang.org/favicon.ico 500.5w`, `,foo/,`, @@ -118,7 +118,7 @@ func TestTypedContent(t *testing.T) { `Hello, World &tc!`, ` dir="ltr"`, `c && alert("Hello, World!");`, - `Hello, World & O'Reilly\x21`, + `Hello, World & O'Reilly\u0021`, `greeting=H%69,&addressee=(World)`, `greeting=H%69,&addressee=(World) 2x, https://golang.org/favicon.ico 500.5w`, `,foo/,`, @@ -133,7 +133,7 @@ func TestTypedContent(t *testing.T) { `Hello, <b>World</b> &tc!`, ` dir="ltr"`, `c && alert("Hello, World!");`, - `Hello, World & O'Reilly\x21`, + `Hello, World & O'Reilly\u0021`, `greeting=H%69,&addressee=(World)`, `greeting=H%69,&addressee=(World) 2x, https://golang.org/favicon.ico 500.5w`, `,foo/,`, @@ -149,7 +149,7 @@ func TestTypedContent(t *testing.T) { // Not escaped. `c && alert("Hello, World!");`, // Escape sequence not over-escaped. - `"Hello, World & O'Reilly\x21"`, + `"Hello, World & O'Reilly\u0021"`, `"greeting=H%69,\u0026addressee=(World)"`, `"greeting=H%69,\u0026addressee=(World) 2x, https://golang.org/favicon.ico 500.5w"`, `",foo/,"`, @@ -165,7 +165,7 @@ func TestTypedContent(t *testing.T) { // Not JS escaped but HTML escaped. `c && alert("Hello, World!");`, // Escape sequence not over-escaped. - `"Hello, World & O'Reilly\x21"`, + `"Hello, World & O'Reilly\u0021"`, `"greeting=H%69,\u0026addressee=(World)"`, `"greeting=H%69,\u0026addressee=(World) 2x, https://golang.org/favicon.ico 500.5w"`, `",foo/,"`, @@ -174,30 +174,30 @@ func TestTypedContent(t *testing.T) { { `<script>alert("{{.}}")</script>`, []string{ - `\x3cb\x3e \x22foo%\x22 O\x27Reilly \x26bar;`, - `a[href =~ \x22\/\/example.com\x22]#foo`, - `Hello, \x3cb\x3eWorld\x3c\/b\x3e \x26amp;tc!`, - ` dir=\x22ltr\x22`, - `c \x26\x26 alert(\x22Hello, World!\x22);`, + `\u003cb\u003e \u0022foo%\u0022 O\u0027Reilly \u0026bar;`, + `a[href =~ \u0022\/\/example.com\u0022]#foo`, + `Hello, \u003cb\u003eWorld\u003c\/b\u003e \u0026amp;tc!`, + ` dir=\u0022ltr\u0022`, + `c \u0026\u0026 alert(\u0022Hello, World!\u0022);`, // Escape sequence not over-escaped. - `Hello, World \x26 O\x27Reilly\x21`, - `greeting=H%69,\x26addressee=(World)`, - `greeting=H%69,\x26addressee=(World) 2x, https:\/\/golang.org\/favicon.ico 500.5w`, + `Hello, World \u0026 O\u0027Reilly\u0021`, + `greeting=H%69,\u0026addressee=(World)`, + `greeting=H%69,\u0026addressee=(World) 2x, https:\/\/golang.org\/favicon.ico 500.5w`, `,foo\/,`, }, }, { `<script type="text/javascript">alert("{{.}}")</script>`, []string{ - `\x3cb\x3e \x22foo%\x22 O\x27Reilly \x26bar;`, - `a[href =~ \x22\/\/example.com\x22]#foo`, - `Hello, \x3cb\x3eWorld\x3c\/b\x3e \x26amp;tc!`, - ` dir=\x22ltr\x22`, - `c \x26\x26 alert(\x22Hello, World!\x22);`, + `\u003cb\u003e \u0022foo%\u0022 O\u0027Reilly \u0026bar;`, + `a[href =~ \u0022\/\/example.com\u0022]#foo`, + `Hello, \u003cb\u003eWorld\u003c\/b\u003e \u0026amp;tc!`, + ` dir=\u0022ltr\u0022`, + `c \u0026\u0026 alert(\u0022Hello, World!\u0022);`, // Escape sequence not over-escaped. - `Hello, World \x26 O\x27Reilly\x21`, - `greeting=H%69,\x26addressee=(World)`, - `greeting=H%69,\x26addressee=(World) 2x, https:\/\/golang.org\/favicon.ico 500.5w`, + `Hello, World \u0026 O\u0027Reilly\u0021`, + `greeting=H%69,\u0026addressee=(World)`, + `greeting=H%69,\u0026addressee=(World) 2x, https:\/\/golang.org\/favicon.ico 500.5w`, `,foo\/,`, }, }, @@ -211,7 +211,7 @@ func TestTypedContent(t *testing.T) { // Not escaped. `c && alert("Hello, World!");`, // Escape sequence not over-escaped. - `"Hello, World & O'Reilly\x21"`, + `"Hello, World & O'Reilly\u0021"`, `"greeting=H%69,\u0026addressee=(World)"`, `"greeting=H%69,\u0026addressee=(World) 2x, https://golang.org/favicon.ico 500.5w"`, `",foo/,"`, @@ -227,7 +227,7 @@ func TestTypedContent(t *testing.T) { `Hello, <b>World</b> &tc!`, ` dir="ltr"`, `c && alert("Hello, World!");`, - `Hello, World & O'Reilly\x21`, + `Hello, World & O'Reilly\u0021`, `greeting=H%69,&addressee=(World)`, `greeting=H%69,&addressee=(World) 2x, https://golang.org/favicon.ico 500.5w`, `,foo/,`, @@ -236,15 +236,15 @@ func TestTypedContent(t *testing.T) { { `<button onclick='alert("{{.}}")'>`, []string{ - `\x3cb\x3e \x22foo%\x22 O\x27Reilly \x26bar;`, - `a[href =~ \x22\/\/example.com\x22]#foo`, - `Hello, \x3cb\x3eWorld\x3c\/b\x3e \x26amp;tc!`, - ` dir=\x22ltr\x22`, - `c \x26\x26 alert(\x22Hello, World!\x22);`, + `\u003cb\u003e \u0022foo%\u0022 O\u0027Reilly \u0026bar;`, + `a[href =~ \u0022\/\/example.com\u0022]#foo`, + `Hello, \u003cb\u003eWorld\u003c\/b\u003e \u0026amp;tc!`, + ` dir=\u0022ltr\u0022`, + `c \u0026\u0026 alert(\u0022Hello, World!\u0022);`, // Escape sequence not over-escaped. - `Hello, World \x26 O\x27Reilly\x21`, - `greeting=H%69,\x26addressee=(World)`, - `greeting=H%69,\x26addressee=(World) 2x, https:\/\/golang.org\/favicon.ico 500.5w`, + `Hello, World \u0026 O\u0027Reilly\u0021`, + `greeting=H%69,\u0026addressee=(World)`, + `greeting=H%69,\u0026addressee=(World) 2x, https:\/\/golang.org\/favicon.ico 500.5w`, `,foo\/,`, }, }, @@ -256,7 +256,7 @@ func TestTypedContent(t *testing.T) { `Hello%2c%20%3cb%3eWorld%3c%2fb%3e%20%26amp%3btc%21`, `%20dir%3d%22ltr%22`, `c%20%26%26%20alert%28%22Hello%2c%20World%21%22%29%3b`, - `Hello%2c%20World%20%26%20O%27Reilly%5cx21`, + `Hello%2c%20World%20%26%20O%27Reilly%5cu0021`, // Quotes and parens are escaped but %69 is not over-escaped. HTML escaping is done. `greeting=H%69,&addressee=%28World%29`, `greeting%3dH%2569%2c%26addressee%3d%28World%29%202x%2c%20https%3a%2f%2fgolang.org%2ffavicon.ico%20500.5w`, @@ -271,7 +271,7 @@ func TestTypedContent(t *testing.T) { `Hello%2c%20%3cb%3eWorld%3c%2fb%3e%20%26amp%3btc%21`, `%20dir%3d%22ltr%22`, `c%20%26%26%20alert%28%22Hello%2c%20World%21%22%29%3b`, - `Hello%2c%20World%20%26%20O%27Reilly%5cx21`, + `Hello%2c%20World%20%26%20O%27Reilly%5cu0021`, // Quotes and parens are escaped but %69 is not over-escaped. HTML escaping is not done. `greeting=H%69,&addressee=%28World%29`, `greeting%3dH%2569%2c%26addressee%3d%28World%29%202x%2c%20https%3a%2f%2fgolang.org%2ffavicon.ico%20500.5w`, diff --git a/tpl/internal/go_templates/htmltemplate/doc.go b/tpl/internal/go_templates/htmltemplate/doc.go index 0a3004a9b..b6a1504f8 100644 --- a/tpl/internal/go_templates/htmltemplate/doc.go +++ b/tpl/internal/go_templates/htmltemplate/doc.go @@ -73,6 +73,51 @@ functions. For these internal escaping functions, if an action pipeline evaluates to a nil interface value, it is treated as though it were an empty string. +Namespaced and data- attributes + +Attributes with a namespace are treated as if they had no namespace. +Given the excerpt + + <a my:href="{{.}}"></a> + +At parse time the attribute will be treated as if it were just "href". +So at parse time the template becomes: + + <a my:href="{{. | urlescaper | attrescaper}}"></a> + +Similarly to attributes with namespaces, attributes with a "data-" prefix are +treated as if they had no "data-" prefix. So given + + <a data-href="{{.}}"></a> + +At parse time this becomes + + <a data-href="{{. | urlescaper | attrescaper}}"></a> + +If an attribute has both a namespace and a "data-" prefix, only the namespace +will be removed when determining the context. For example + + <a my:data-href="{{.}}"></a> + +This is handled as if "my:data-href" was just "data-href" and not "href" as +it would be if the "data-" prefix were to be ignored too. Thus at parse +time this becomes just + + <a my:data-href="{{. | attrescaper}}"></a> + +As a special case, attributes with the namespace "xmlns" are always treated +as containing URLs. Given the excerpts + + <a xmlns:title="{{.}}"></a> + <a xmlns:href="{{.}}"></a> + <a xmlns:onclick="{{.}}"></a> + +At parse time they become: + + <a xmlns:title="{{. | urlescaper | attrescaper}}"></a> + <a xmlns:href="{{. | urlescaper | attrescaper}}"></a> + <a xmlns:onclick="{{. | urlescaper | attrescaper}}"></a> + Errors See the documentation of ErrorCode for details. diff --git a/tpl/internal/go_templates/htmltemplate/escape_test.go b/tpl/internal/go_templates/htmltemplate/escape_test.go index 9e9db7800..075db4e13 100644 --- a/tpl/internal/go_templates/htmltemplate/escape_test.go +++ b/tpl/internal/go_templates/htmltemplate/escape_test.go @@ -242,7 +242,7 @@ func TestEscape(t *testing.T) { { "jsStr", "<button onclick='alert("{{.H}}")'>", - `<button onclick='alert("\x3cHello\x3e")'>`, + `<button onclick='alert("\u003cHello\u003e")'>`, }, { "badMarshaler", @@ -263,7 +263,7 @@ func TestEscape(t *testing.T) { { "jsRe", `<button onclick='alert(/{{"foo+bar"}}/.test(""))'>`, - `<button onclick='alert(/foo\x2bbar/.test(""))'>`, + `<button onclick='alert(/foo\u002bbar/.test(""))'>`, }, { "jsReBlank", @@ -829,7 +829,7 @@ func TestEscapeSet(t *testing.T) { "main": `<button onclick="title='{{template "helper"}}'; ...">{{template "helper"}}</button>`, "helper": `{{11}} of {{"<100>"}}`, }, - `<button onclick="title='11 of \x3c100\x3e'; ...">11 of <100></button>`, + `<button onclick="title='11 of \u003c100\u003e'; ...">11 of <100></button>`, }, // A non-recursive template that ends in a different context. // helper starts in jsCtxRegexp and ends in jsCtxDivOp. diff --git a/tpl/internal/go_templates/htmltemplate/example_test.go b/tpl/internal/go_templates/htmltemplate/example_test.go index a3e7910ee..a93b8d2fb 100644 --- a/tpl/internal/go_templates/htmltemplate/example_test.go +++ b/tpl/internal/go_templates/htmltemplate/example_test.go @@ -119,9 +119,9 @@ func Example_escape() { // "Fran & Freddie's Diner" <[email protected]> // "Fran & Freddie's Diner" <[email protected]> // "Fran & Freddie's Diner"32<[email protected]> - // \"Fran & Freddie\'s Diner\" \[email protected]\x3E - // \"Fran & Freddie\'s Diner\" \[email protected]\x3E - // \"Fran & Freddie\'s Diner\"32\[email protected]\x3E + // \"Fran \u0026 Freddie\'s Diner\" \[email protected]\u003E + // \"Fran \u0026 Freddie\'s Diner\" \[email protected]\u003E + // \"Fran \u0026 Freddie\'s Diner\"32\[email protected]\u003E // %22Fran+%26+Freddie%27s+Diner%2232%3Ctasty%40example.com%3E } diff --git a/tpl/internal/go_templates/htmltemplate/js.go b/tpl/internal/go_templates/htmltemplate/js.go index 57622d152..cfd413461 100644 --- a/tpl/internal/go_templates/htmltemplate/js.go +++ b/tpl/internal/go_templates/htmltemplate/js.go @@ -164,7 +164,6 @@ func jsValEscaper(args ...interface{}) string { } // TODO: detect cycles before calling Marshal which loops infinitely on // cyclic data. This may be an unacceptable DoS risk. - b, err := json.Marshal(a) if err != nil { // Put a space before comment so that if it is flush against @@ -179,8 +178,8 @@ func jsValEscaper(args ...interface{}) string { // TODO: maybe post-process output to prevent it from containing // "<!--", "-->", "<![CDATA[", "]]>", or "</script" // in case custom marshalers produce output containing those. - - // TODO: Maybe abbreviate \u00ab to \xab to produce more compact output. + // Note: Do not use \x escaping to save bytes because it is not JSON compatible and this escaper + // supports ld+json content-type. if len(b) == 0 { // In, `x=y/{{.}}*z` a json.Marshaler that produces "" should // not cause the output `x=y/*z`. @@ -261,6 +260,8 @@ func replace(s string, replacementTable []string) string { r, w = utf8.DecodeRuneInString(s[i:]) var repl string switch { + case int(r) < len(lowUnicodeReplacementTable): + repl = lowUnicodeReplacementTable[r] case int(r) < len(replacementTable) && replacementTable[r] != "": repl = replacementTable[r] case r == '\u2028': @@ -284,67 +285,80 @@ func replace(s string, replacementTable []string) string { return b.String() } +var lowUnicodeReplacementTable = []string{ + 0: `\u0000`, 1: `\u0001`, 2: `\u0002`, 3: `\u0003`, 4: `\u0004`, 5: `\u0005`, 6: `\u0006`, + '\a': `\u0007`, + '\b': `\u0008`, + '\t': `\t`, + '\n': `\n`, + '\v': `\u000b`, // "\v" == "v" on IE 6. + '\f': `\f`, + '\r': `\r`, + 0xe: `\u000e`, 0xf: `\u000f`, 0x10: `\u0010`, 0x11: `\u0011`, 0x12: `\u0012`, 0x13: `\u0013`, + 0x14: `\u0014`, 0x15: `\u0015`, 0x16: `\u0016`, 0x17: `\u0017`, 0x18: `\u0018`, 0x19: `\u0019`, + 0x1a: `\u001a`, 0x1b: `\u001b`, 0x1c: `\u001c`, 0x1d: `\u001d`, 0x1e: `\u001e`, 0x1f: `\u001f`, +} + var jsStrReplacementTable = []string{ - 0: `\0`, + 0: `\u0000`, '\t': `\t`, '\n': `\n`, - '\v': `\x0b`, // "\v" == "v" on IE 6. + '\v': `\u000b`, // "\v" == "v" on IE 6. '\f': `\f`, '\r': `\r`, // Encode HTML specials as hex so the output can be embedded // in HTML attributes without further encoding. - '"': `\x22`, - '&': `\x26`, - '\'': `\x27`, - '+': `\x2b`, + '"': `\u0022`, + '&': `\u0026`, + '\'': `\u0027`, + '+': `\u002b`, '/': `\/`, - '<': `\x3c`, - '>': `\x3e`, + '<': `\u003c`, + '>': `\u003e`, '\\': `\\`, } // jsStrNormReplacementTable is like jsStrReplacementTable but does not // overencode existing escapes since this table has no entry for `\`. var jsStrNormReplacementTable = []string{ - 0: `\0`, + 0: `\u0000`, '\t': `\t`, '\n': `\n`, - '\v': `\x0b`, // "\v" == "v" on IE 6. + '\v': `\u000b`, // "\v" == "v" on IE 6. '\f': `\f`, '\r': `\r`, // Encode HTML specials as hex so the output can be embedded // in HTML attributes without further encoding. - '"': `\x22`, - '&': `\x26`, - '\'': `\x27`, - '+': `\x2b`, + '"': `\u0022`, + '&': `\u0026`, + '\'': `\u0027`, + '+': `\u002b`, '/': `\/`, - '<': `\x3c`, - '>': `\x3e`, + '<': `\u003c`, + '>': `\u003e`, } - var jsRegexpReplacementTable = []string{ - 0: `\0`, + 0: `\u0000`, '\t': `\t`, '\n': `\n`, - '\v': `\x0b`, // "\v" == "v" on IE 6. + '\v': `\u000b`, // "\v" == "v" on IE 6. '\f': `\f`, '\r': `\r`, // Encode HTML specials as hex so the output can be embedded // in HTML attributes without further encoding. - '"': `\x22`, + '"': `\u0022`, '$': `\$`, - '&': `\x26`, - '\'': `\x27`, + '&': `\u0026`, + '\'': `\u0027`, '(': `\(`, ')': `\)`, '*': `\*`, - '+': `\x2b`, + '+': `\u002b`, '-': `\-`, '.': `\.`, '/': `\/`, - '<': `\x3c`, - '>': `\x3e`, + '<': `\u003c`, + '>': `\u003e`, '?': `\?`, '[': `\[`, '\\': `\\`, @@ -384,11 +398,11 @@ func isJSType(mimeType string) bool { // https://tools.ietf.org/html/rfc7231#section-3.1.1 // https://tools.ietf.org/html/rfc4329#section-3 // https://www.ietf.org/rfc/rfc4627.txt - mimeType = strings.ToLower(mimeType) // discard parameters if i := strings.Index(mimeType, ";"); i >= 0 { mimeType = mimeType[:i] } + mimeType = strings.ToLower(mimeType) mimeType = strings.TrimSpace(mimeType) switch mimeType { case diff --git a/tpl/internal/go_templates/htmltemplate/js_test.go b/tpl/internal/go_templates/htmltemplate/js_test.go index 0a6f332a6..e15087f0f 100644 --- a/tpl/internal/go_templates/htmltemplate/js_test.go +++ b/tpl/internal/go_templates/htmltemplate/js_test.go @@ -139,7 +139,7 @@ func TestJSValEscaper(t *testing.T) { {"foo", `"foo"`}, // Newlines. {"\r\n\u2028\u2029", `"\r\n\u2028\u2029"`}, - // "\v" == "v" on IE 6 so use "\x0b" instead. + // "\v" == "v" on IE 6 so use "\u000b" instead. {"\t\x0b", `"\t\u000b"`}, {struct{ X, Y int }{1, 2}, `{"X":1,"Y":2}`}, {[]interface{}{}, "[]"}, @@ -175,7 +175,7 @@ func TestJSStrEscaper(t *testing.T) { }{ {"", ``}, {"foo", `foo`}, - {"\u0000", `\0`}, + {"\u0000", `\u0000`}, {"\t", `\t`}, {"\n", `\n`}, {"\r", `\r`}, @@ -185,14 +185,14 @@ func TestJSStrEscaper(t *testing.T) { {"\\n", `\\n`}, {"foo\r\nbar", `foo\r\nbar`}, // Preserve attribute boundaries. - {`"`, `\x22`}, - {`'`, `\x27`}, + {`"`, `\u0022`}, + {`'`, `\u0027`}, // Allow embedding in HTML without further escaping. - {`&`, `\x26amp;`}, + {`&`, `\u0026amp;`}, // Prevent breaking out of text node and element boundaries. - {"</script>", `\x3c\/script\x3e`}, - {"<![CDATA[", `\x3c![CDATA[`}, - {"]]>", `]]\x3e`}, + {"</script>", `\u003c\/script\u003e`}, + {"<![CDATA[", `\u003c![CDATA[`}, + {"]]>", `]]\u003e`}, // https://dev.w3.org/html5/markup/aria/syntax.html#escaping-text-span // "The text in style, script, title, and textarea elements // must not have an escaping text span start that is not @@ -203,11 +203,11 @@ func TestJSStrEscaper(t *testing.T) { // allow regular text content to be interpreted as script // allowing script execution via a combination of a JS string // injection followed by an HTML text injection. - {"<!--", `\x3c!--`}, - {"-->", `--\x3e`}, + {"<!--", `\u003c!--`}, + {"-->", `--\u003e`}, // From https://code.google.com/p/doctype/wiki/ArticleUtf7 {"+ADw-script+AD4-alert(1)+ADw-/script+AD4-", - `\x2bADw-script\x2bAD4-alert(1)\x2bADw-\/script\x2bAD4-`, + `\u002bADw-script\u002bAD4-alert(1)\u002bADw-\/script\u002bAD4-`, }, // Invalid UTF-8 sequence {"foo\xA0bar", "foo\xA0bar"}, @@ -230,7 +230,7 @@ func TestJSRegexpEscaper(t *testing.T) { }{ {"", `(?:)`}, {"foo", `foo`}, - {"\u0000", `\0`}, + {"\u0000", `\u0000`}, {"\t", `\t`}, {"\n", `\n`}, {"\r", `\r`}, @@ -240,19 +240,19 @@ func TestJSRegexpEscaper(t *testing.T) { {"\\n", `\\n`}, {"foo\r\nbar", `foo\r\nbar`}, // Preserve attribute boundaries. - {`"`, `\x22`}, - {`'`, `\x27`}, + {`"`, `\u0022`}, + {`'`, `\u0027`}, // Allow embedding in HTML without further escaping. - {`&`, `\x26amp;`}, + {`&`, `\u0026amp;`}, // Prevent breaking out of text node and element boundaries. - {"</script>", `\x3c\/script\x3e`}, - {"<![CDATA[", `\x3c!\[CDATA\[`}, - {"]]>", `\]\]\x3e`}, + {"</script>", `\u003c\/script\u003e`}, + {"<![CDATA[", `\u003c!\[CDATA\[`}, + {"]]>", `\]\]\u003e`}, // Escaping text spans. - {"<!--", `\x3c!\-\-`}, - {"-->", `\-\-\x3e`}, + {"<!--", `\u003c!\-\-`}, + {"-->", `\-\-\u003e`}, {"*", `\*`}, - {"+", `\x2b`}, + {"+", `\u002b`}, {"?", `\?`}, {"[](){}", `\[\]\(\)\{\}`}, {"$foo|x.y", `\$foo\|x\.y`}, @@ -286,27 +286,27 @@ func TestEscapersOnLower7AndSelectHighCodepoints(t *testing.T) { { "jsStrEscaper", jsStrEscaper, - "\\0\x01\x02\x03\x04\x05\x06\x07" + - "\x08\\t\\n\\x0b\\f\\r\x0E\x0F" + - "\x10\x11\x12\x13\x14\x15\x16\x17" + - "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + - ` !\x22#$%\x26\x27()*\x2b,-.\/` + - `0123456789:;\x3c=\x3e?` + + `\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007` + + `\u0008\t\n\u000b\f\r\u000e\u000f` + + `\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017` + + `\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f` + + ` !\u0022#$%\u0026\u0027()*\u002b,-.\/` + + `0123456789:;\u003c=\u003e?` + `@ABCDEFGHIJKLMNO` + `PQRSTUVWXYZ[\\]^_` + "`abcdefghijklmno" + - "pqrstuvwxyz{|}~\x7f" + + "pqrstuvwxyz{|}~\u007f" + "\u00A0\u0100\\u2028\\u2029\ufeff\U0001D11E", }, { "jsRegexpEscaper", jsRegexpEscaper, - "\\0\x01\x02\x03\x04\x05\x06\x07" + - "\x08\\t\\n\\x0b\\f\\r\x0E\x0F" + - "\x10\x11\x12\x13\x14\x15\x16\x17" + - "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + - ` !\x22#\$%\x26\x27\(\)\*\x2b,\-\.\/` + - `0123456789:;\x3c=\x3e\?` + + `\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007` + + `\u0008\t\n\u000b\f\r\u000e\u000f` + + `\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017` + + `\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f` + + ` !\u0022#\$%\u0026\u0027\(\)\*\u002b,\-\.\/` + + `0123456789:;\u003c=\u003e\?` + `@ABCDEFGHIJKLMNO` + `PQRSTUVWXYZ\[\\\]\^_` + "`abcdefghijklmno" + diff --git a/tpl/internal/go_templates/htmltemplate/template_test.go b/tpl/internal/go_templates/htmltemplate/template_test.go index 8adf3324b..589e6912a 100644 --- a/tpl/internal/go_templates/htmltemplate/template_test.go +++ b/tpl/internal/go_templates/htmltemplate/template_test.go @@ -8,6 +8,7 @@ package template_test import ( "bytes" + "encoding/json" "strings" "testing" @@ -124,6 +125,44 @@ func TestNumbers(t *testing.T) { c.mustExecute(c.root, nil, "12.34 7.5") } +func TestStringsInScriptsWithJsonContentTypeAreCorrectlyEscaped(t *testing.T) { + // See #33671 and #37634 for more context on this. + tests := []struct{ name, in string }{ + {"empty", ""}, + {"invalid", string(rune(-1))}, + {"null", "\u0000"}, + {"unit separator", "\u001F"}, + {"tab", "\t"}, + {"gt and lt", "<>"}, + {"quotes", `'"`}, + {"ASCII letters", "ASCII letters"}, + {"Unicode", "ʕ⊙ϖ⊙ʔ"}, + {"Pizza", "🍕"}, + } + const ( + prefix = `<script type="application/ld+json">` + suffix = `</script>` + templ = prefix + `"{{.}}"` + suffix + ) + tpl := Must(New("JS string is JSON string").Parse(templ)) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + if err := tpl.Execute(&buf, tt.in); err != nil { + t.Fatalf("Cannot render template: %v", err) + } + trimmed := bytes.TrimSuffix(bytes.TrimPrefix(buf.Bytes(), []byte(prefix)), []byte(suffix)) + var got string + if err := json.Unmarshal(trimmed, &got); err != nil { + t.Fatalf("Cannot parse JS string %q as JSON: %v", trimmed[1:len(trimmed)-1], err) + } + if got != tt.in { + t.Errorf("Serialization changed the string value: got %q want %q", got, tt.in) + } + }) + } +} + type testCase struct { t *testing.T root *Template diff --git a/tpl/internal/go_templates/testenv/testenv.go b/tpl/internal/go_templates/testenv/testenv.go new file mode 100644 index 000000000..90044570d --- /dev/null +++ b/tpl/internal/go_templates/testenv/testenv.go @@ -0,0 +1,272 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package testenv provides information about what functionality +// is available in different testing environments run by the Go team. +// +// It is an internal package because these details are specific +// to the Go team's test setup (on build.golang.org) and not +// fundamental to tests in general. +package testenv + +import ( + "errors" + "flag" + "github.com/gohugoio/hugo/tpl/internal/go_templates/cfg" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "testing" +) + +// Builder reports the name of the builder running this test +// (for example, "linux-amd64" or "windows-386-gce"). +// If the test is not running on the build infrastructure, +// Builder returns the empty string. +func Builder() string { + return os.Getenv("GO_BUILDER_NAME") +} + +// HasGoBuild reports whether the current system can build programs with ``go build'' +// and then run them with os.StartProcess or exec.Command. +func HasGoBuild() bool { + if os.Getenv("GO_GCFLAGS") != "" { + // It's too much work to require every caller of the go command + // to pass along "-gcflags="+os.Getenv("GO_GCFLAGS"). + // For now, if $GO_GCFLAGS is set, report that we simply can't + // run go build. + return false + } + switch runtime.GOOS { + case "android", "js": + return false + case "darwin": + if runtime.GOARCH == "arm64" { + return false + } + } + return true +} + +// MustHaveGoBuild checks that the current system can build programs with ``go build'' +// and then run them with os.StartProcess or exec.Command. +// If not, MustHaveGoBuild calls t.Skip with an explanation. +func MustHaveGoBuild(t testing.TB) { + if os.Getenv("GO_GCFLAGS") != "" { + t.Skipf("skipping test: 'go build' not compatible with setting $GO_GCFLAGS") + } + if !HasGoBuild() { + t.Skipf("skipping test: 'go build' not available on %s/%s", runtime.GOOS, runtime.GOARCH) + } +} + +// HasGoRun reports whether the current system can run programs with ``go run.'' +func HasGoRun() bool { + // For now, having go run and having go build are the same. + return HasGoBuild() +} + +// MustHaveGoRun checks that the current system can run programs with ``go run.'' +// If not, MustHaveGoRun calls t.Skip with an explanation. +func MustHaveGoRun(t testing.TB) { + if !HasGoRun() { + t.Skipf("skipping test: 'go run' not available on %s/%s", runtime.GOOS, runtime.GOARCH) + } +} + +// GoToolPath reports the path to the Go tool. +// It is a convenience wrapper around GoTool. +// If the tool is unavailable GoToolPath calls t.Skip. +// If the tool should be available and isn't, GoToolPath calls t.Fatal. +func GoToolPath(t testing.TB) string { + MustHaveGoBuild(t) + path, err := GoTool() + if err != nil { + t.Fatal(err) + } + // Add all environment variables that affect the Go command to test metadata. + // Cached test results will be invalidate when these variables change. + // See golang.org/issue/32285. + for _, envVar := range strings.Fields(cfg.KnownEnv) { + os.Getenv(envVar) + } + return path +} + +// GoTool reports the path to the Go tool. +func GoTool() (string, error) { + if !HasGoBuild() { + return "", errors.New("platform cannot run go tool") + } + var exeSuffix string + if runtime.GOOS == "windows" { + exeSuffix = ".exe" + } + path := filepath.Join(runtime.GOROOT(), "bin", "go"+exeSuffix) + if _, err := os.Stat(path); err == nil { + return path, nil + } + goBin, err := exec.LookPath("go" + exeSuffix) + if err != nil { + return "", errors.New("cannot find go tool: " + err.Error()) + } + return goBin, nil +} + +// HasExec reports whether the current system can start new processes +// using os.StartProcess or (more commonly) exec.Command. +func HasExec() bool { + switch runtime.GOOS { + case "js": + return false + case "darwin": + if runtime.GOARCH == "arm64" { + return false + } + } + return true +} + +// HasSrc reports whether the entire source tree is available under GOROOT. +func HasSrc() bool { + switch runtime.GOOS { + case "darwin": + if runtime.GOARCH == "arm64" { + return false + } + } + return true +} + +// MustHaveExec checks that the current system can start new processes +// using os.StartProcess or (more commonly) exec.Command. +// If not, MustHaveExec calls t.Skip with an explanation. +func MustHaveExec(t testing.TB) { + if !HasExec() { + t.Skipf("skipping test: cannot exec subprocess on %s/%s", runtime.GOOS, runtime.GOARCH) + } +} + +var execPaths sync.Map // path -> error + +// MustHaveExecPath checks that the current system can start the named executable +// using os.StartProcess or (more commonly) exec.Command. +// If not, MustHaveExecPath calls t.Skip with an explanation. +func MustHaveExecPath(t testing.TB, path string) { + MustHaveExec(t) + + err, found := execPaths.Load(path) + if !found { + _, err = exec.LookPath(path) + err, _ = execPaths.LoadOrStore(path, err) + } + if err != nil { + t.Skipf("skipping test: %s: %s", path, err) + } +} + +// HasExternalNetwork reports whether the current system can use +// external (non-localhost) networks. +func HasExternalNetwork() bool { + return !testing.Short() && runtime.GOOS != "js" +} + +// MustHaveExternalNetwork checks that the current system can use +// external (non-localhost) networks. +// If not, MustHaveExternalNetwork calls t.Skip with an explanation. +func MustHaveExternalNetwork(t testing.TB) { + if runtime.GOOS == "js" { + t.Skipf("skipping test: no external network on %s", runtime.GOOS) + } + if testing.Short() { + t.Skipf("skipping test: no external network in -short mode") + } +} + +var haveCGO bool + +// HasCGO reports whether the current system can use cgo. +func HasCGO() bool { + return haveCGO +} + +// MustHaveCGO calls t.Skip if cgo is not available. +func MustHaveCGO(t testing.TB) { + if !haveCGO { + t.Skipf("skipping test: no cgo") + } +} + +// HasSymlink reports whether the current system can use os.Symlink. +func HasSymlink() bool { + ok, _ := hasSymlink() + return ok +} + +// MustHaveSymlink reports whether the current system can use os.Symlink. +// If not, MustHaveSymlink calls t.Skip with an explanation. +func MustHaveSymlink(t testing.TB) { + ok, reason := hasSymlink() + if !ok { + t.Skipf("skipping test: cannot make symlinks on %s/%s%s", runtime.GOOS, runtime.GOARCH, reason) + } +} + +// HasLink reports whether the current system can use os.Link. +func HasLink() bool { + // From Android release M (Marshmallow), hard linking files is blocked + // and an attempt to call link() on a file will return EACCES. + // - https://code.google.com/p/android-developer-preview/issues/detail?id=3150 + return runtime.GOOS != "plan9" && runtime.GOOS != "android" +} + +// MustHaveLink reports whether the current system can use os.Link. +// If not, MustHaveLink calls t.Skip with an explanation. +func MustHaveLink(t testing.TB) { + if !HasLink() { + t.Skipf("skipping test: hardlinks are not supported on %s/%s", runtime.GOOS, runtime.GOARCH) + } +} + +var flaky = flag.Bool("flaky", false, "run known-flaky tests too") + +func SkipFlaky(t testing.TB, issue int) { + t.Helper() + if !*flaky { + t.Skipf("skipping known flaky test without the -flaky flag; see golang.org/issue/%d", issue) + } +} + +func SkipFlakyNet(t testing.TB) { + t.Helper() + if v, _ := strconv.ParseBool(os.Getenv("GO_BUILDER_FLAKY_NET")); v { + t.Skip("skipping test on builder known to have frequent network failures") + } +} + +// CleanCmdEnv will fill cmd.Env with the environment, excluding certain +// variables that could modify the behavior of the Go tools such as +// GODEBUG and GOTRACEBACK. +func CleanCmdEnv(cmd *exec.Cmd) *exec.Cmd { + if cmd.Env != nil { + panic("environment already set") + } + for _, env := range os.Environ() { + // Exclude GODEBUG from the environment to prevent its output + // from breaking tests that are trying to parse other command output. + if strings.HasPrefix(env, "GODEBUG=") { + continue + } + // Exclude GOTRACEBACK for the same reason. + if strings.HasPrefix(env, "GOTRACEBACK=") { + continue + } + cmd.Env = append(cmd.Env, env) + } + return cmd +} diff --git a/tpl/internal/go_templates/testenv/testenv_cgo.go b/tpl/internal/go_templates/testenv/testenv_cgo.go new file mode 100644 index 000000000..e3d4d16b3 --- /dev/null +++ b/tpl/internal/go_templates/testenv/testenv_cgo.go @@ -0,0 +1,11 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo + +package testenv + +func init() { + haveCGO = true +} diff --git a/tpl/internal/go_templates/testenv/testenv_notwin.go b/tpl/internal/go_templates/testenv/testenv_notwin.go new file mode 100644 index 000000000..ccb5d5585 --- /dev/null +++ b/tpl/internal/go_templates/testenv/testenv_notwin.go @@ -0,0 +1,20 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !windows + +package testenv + +import ( + "runtime" +) + +func hasSymlink() (ok bool, reason string) { + switch runtime.GOOS { + case "android", "plan9": + return false, "" + } + + return true, "" +} diff --git a/tpl/internal/go_templates/testenv/testenv_windows.go b/tpl/internal/go_templates/testenv/testenv_windows.go new file mode 100644 index 000000000..eb8d6ac16 --- /dev/null +++ b/tpl/internal/go_templates/testenv/testenv_windows.go @@ -0,0 +1,48 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testenv + +import ( + "io/ioutil" + "os" + "path/filepath" + "sync" + "syscall" +) + +var symlinkOnce sync.Once +var winSymlinkErr error + +func initWinHasSymlink() { + tmpdir, err := ioutil.TempDir("", "symtest") + if err != nil { + panic("failed to create temp directory: " + err.Error()) + } + defer os.RemoveAll(tmpdir) + + err = os.Symlink("target", filepath.Join(tmpdir, "symlink")) + if err != nil { + err = err.(*os.LinkError).Err + switch err { + case syscall.EWINDOWS, syscall.ERROR_PRIVILEGE_NOT_HELD: + winSymlinkErr = err + } + } +} + +func hasSymlink() (ok bool, reason string) { + symlinkOnce.Do(initWinHasSymlink) + + switch winSymlinkErr { + case nil: + return true, "" + case syscall.EWINDOWS: + return false, ": symlinks are not supported on your version of Windows" + case syscall.ERROR_PRIVILEGE_NOT_HELD: + return false, ": you don't have enough privileges to create symlinks" + } + + return false, "" +} diff --git a/tpl/internal/go_templates/texttemplate/doc.go b/tpl/internal/go_templates/texttemplate/doc.go index dbffaa495..4b0efd2df 100644 --- a/tpl/internal/go_templates/texttemplate/doc.go +++ b/tpl/internal/go_templates/texttemplate/doc.go @@ -102,8 +102,8 @@ data, defined in detail in the corresponding sections that follow. If the value of the pipeline has length zero, nothing is output; otherwise, dot is set to the successive elements of the array, slice, or map and T1 is executed. If the value is a map and the - keys are of basic type with a defined order ("comparable"), the - elements will be visited in sorted key order. + keys are of basic type with a defined order, the elements will be + visited in sorted key order. {{range pipeline}} T1 {{else}} T0 {{end}} The value of the pipeline must be an array, slice, map, or channel. @@ -385,14 +385,12 @@ returning in effect (Unlike with || in Go, however, eq is a function call and all the arguments will be evaluated.) -The comparison functions work on basic types only (or named basic -types, such as "type Celsius float32"). They implement the Go rules -for comparison of values, except that size and exact type are -ignored, so any integer value, signed or unsigned, may be compared -with any other integer value. (The arithmetic value is compared, -not the bit pattern, so all negative integers are less than all -unsigned integers.) However, as usual, one may not compare an int -with a float32 and so on. +The comparison functions work on any values whose type Go defines as +comparable. For basic types such as integers, the rules are relaxed: +size and exact type are ignored, so any integer value, signed or unsigned, +may be compared with any other integer value. (The arithmetic value is compared, +not the bit pattern, so all negative integers are less than all unsigned integers.) +However, as usual, one may not compare an int with a float32 and so on. Associated templates diff --git a/tpl/internal/go_templates/texttemplate/exec.go b/tpl/internal/go_templates/texttemplate/exec.go index f400abd9c..879cd0884 100644 --- a/tpl/internal/go_templates/texttemplate/exec.go +++ b/tpl/internal/go_templates/texttemplate/exec.go @@ -5,7 +5,6 @@ package template import ( - "bytes" "fmt" "github.com/gohugoio/hugo/tpl/internal/go_templates/fmtsort" "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse" @@ -230,21 +229,19 @@ func (t *Template) DefinedTemplates() string { if t.common == nil { return "" } - var b bytes.Buffer + var b strings.Builder for name, tmpl := range t.tmpl { if tmpl.Tree == nil || tmpl.Root == nil { continue } - if b.Len() > 0 { + if b.Len() == 0 { + b.WriteString("; defined templates are: ") + } else { b.WriteString(", ") } fmt.Fprintf(&b, "%q", name) } - var s string - if b.Len() > 0 { - s = "; defined templates are: " + b.String() - } - return s + return b.String() } // Walk functions step through the major pieces of the template structure, @@ -464,7 +461,8 @@ func (s *state) evalCommand(dot reflect.Value, cmd *parse.CommandNode, final ref // Must be a function. return s.evalFunction(dot, n, cmd, cmd.Args, final) case *parse.PipeNode: - // Parenthesized pipeline. The arguments are all inside the pipeline; final is ignored. + // Parenthesized pipeline. The arguments are all inside the pipeline; final must be absent. + s.notAFunction(cmd.Args, final) return s.evalPipeline(dot, n) case *parse.VariableNode: return s.evalVariableNode(dot, n, cmd.Args, final) @@ -499,20 +497,29 @@ func (s *state) idealConstant(constant *parse.NumberNode) reflect.Value { switch { case constant.IsComplex: return reflect.ValueOf(constant.Complex128) // incontrovertible. - case constant.IsFloat && !isHexInt(constant.Text) && strings.ContainsAny(constant.Text, ".eEpP"): + + case constant.IsFloat && + !isHexInt(constant.Text) && !isRuneInt(constant.Text) && + strings.ContainsAny(constant.Text, ".eEpP"): return reflect.ValueOf(constant.Float64) + case constant.IsInt: n := int(constant.Int64) if int64(n) != constant.Int64 { s.errorf("%s overflows int", constant.Text) } return reflect.ValueOf(n) + case constant.IsUint: s.errorf("%s overflows int", constant.Text) } return zero } +func isRuneInt(s string) bool { + return len(s) > 0 && s[0] == '\'' +} + func isHexInt(s string) bool { return len(s) > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X') && !strings.ContainsAny(s, "pP") } diff --git a/tpl/internal/go_templates/texttemplate/exec_test.go b/tpl/internal/go_templates/texttemplate/exec_test.go index 504967db6..940a1de6a 100644 --- a/tpl/internal/go_templates/texttemplate/exec_test.go +++ b/tpl/internal/go_templates/texttemplate/exec_test.go @@ -354,6 +354,12 @@ var execTests = []execTest{ {"field on interface", "{{.foo}}", "<no value>", nil, true}, {"field on parenthesized interface", "{{(.).foo}}", "<no value>", nil, true}, + // Issue 31810: Parenthesized first element of pipeline with arguments. + // See also TestIssue31810. + {"unparenthesized non-function", "{{1 2}}", "", nil, false}, + {"parenthesized non-function", "{{(1) 2}}", "", nil, false}, + {"parenthesized non-function with no args", "{{(1)}}", "1", nil, true}, // This is fine. + // Method calls. {".Method0", "-{{.Method0}}-", "-M0-", tVal, true}, {".Method1(1234)", "-{{.Method1 1234}}-", "-1234-", tVal, true}, @@ -498,6 +504,7 @@ var execTests = []execTest{ {"map MUI64S", "{{index .MUI64S 3}}", "ui643", tVal, true}, {"map MI8S", "{{index .MI8S 3}}", "i83", tVal, true}, {"map MUI8S", "{{index .MUI8S 2}}", "u82", tVal, true}, + {"index of an interface field", "{{index .Empty3 0}}", "7", tVal, true}, // Slicing. {"slice[:]", "{{slice .SI}}", "[3 4 5]", tVal, true}, @@ -523,12 +530,14 @@ var execTests = []execTest{ {"string[1:2]", "{{slice .S 1 2}}", "y", tVal, true}, {"out of range", "{{slice .S 1 5}}", "", tVal, false}, {"3-index slice of string", "{{slice .S 1 2 2}}", "", tVal, false}, + {"slice of an interface field", "{{slice .Empty3 0 1}}", "[7]", tVal, true}, // Len. {"slice", "{{len .SI}}", "3", tVal, true}, {"map", "{{len .MSI }}", "3", tVal, true}, {"len of int", "{{len 3}}", "", tVal, false}, {"len of nothing", "{{len .Empty0}}", "", tVal, false}, + {"len of an interface field", "{{len .Empty3}}", "2", tVal, true}, // With. {"with true", "{{with true}}{{.}}{{end}}", "true", tVal, true}, @@ -665,6 +674,12 @@ var execTests = []execTest{ {"bug17c", "{{len .NonEmptyInterfacePtS}}", "2", tVal, true}, {"bug17d", "{{index .NonEmptyInterfacePtS 0}}", "a", tVal, true}, {"bug17e", "{{range .NonEmptyInterfacePtS}}-{{.}}-{{end}}", "-a--b-", tVal, true}, + + // More variadic function corner cases. Some runes would get evaluated + // as constant floats instead of ints. Issue 34483. + {"bug18a", "{{eq . '.'}}", "true", '.', true}, + {"bug18b", "{{eq . 'e'}}", "true", 'e', true}, + {"bug18c", "{{eq . 'P'}}", "true", 'P', true}, } func zeroArgs() string { @@ -898,7 +913,9 @@ func TestJSEscaping(t *testing.T) { {`Go "jump" \`, `Go \"jump\" \\`}, {`Yukihiro says "今日は世界"`, `Yukihiro says \"今日は世界\"`}, {"unprintable \uFDFF", `unprintable \uFDFF`}, - {`<html>`, `\x3Chtml\x3E`}, + {`<html>`, `\u003Chtml\u003E`}, + {`no = in attributes`, `no \u003D in attributes`}, + {`' does not become HTML entity`, `\u0026#x27; does not become HTML entity`}, } for _, tc := range testCases { s := JSEscapeString(tc.in) @@ -1158,19 +1175,41 @@ var cmpTests = []cmpTest{ {"ge .Uthree .NegOne", "true", true}, {"eq (index `x` 0) 'x'", "true", true}, // The example that triggered this rule. {"eq (index `x` 0) 'y'", "false", true}, + {"eq .V1 .V2", "true", true}, + {"eq .Ptr .Ptr", "true", true}, + {"eq .Ptr .NilPtr", "false", true}, + {"eq .NilPtr .NilPtr", "true", true}, + {"eq .Iface1 .Iface1", "true", true}, + {"eq .Iface1 .Iface2", "false", true}, + {"eq .Iface2 .Iface2", "true", true}, // Errors - {"eq `xy` 1", "", false}, // Different types. - {"eq 2 2.0", "", false}, // Different types. - {"lt true true", "", false}, // Unordered types. - {"lt 1+0i 1+0i", "", false}, // Unordered types. + {"eq `xy` 1", "", false}, // Different types. + {"eq 2 2.0", "", false}, // Different types. + {"lt true true", "", false}, // Unordered types. + {"lt 1+0i 1+0i", "", false}, // Unordered types. + {"eq .Ptr 1", "", false}, // Incompatible types. + {"eq .Ptr .NegOne", "", false}, // Incompatible types. + {"eq .Map .Map", "", false}, // Uncomparable types. + {"eq .Map .V1", "", false}, // Uncomparable types. } func TestComparison(t *testing.T) { b := new(bytes.Buffer) var cmpStruct = struct { - Uthree, Ufour uint - NegOne, Three int - }{3, 4, -1, 3} + Uthree, Ufour uint + NegOne, Three int + Ptr, NilPtr *int + Map map[int]int + V1, V2 V + Iface1, Iface2 fmt.Stringer + }{ + Uthree: 3, + Ufour: 4, + NegOne: -1, + Three: 3, + Ptr: new(int), + Iface1: b, + } for _, test := range cmpTests { text := fmt.Sprintf("{{if %s}}true{{else}}false{{end}}", test.expr) tmpl, err := New("empty").Parse(text) @@ -1622,3 +1661,41 @@ func TestExecutePanicDuringCall(t *testing.T) { } } } + +// Issue 31810. Check that a parenthesized first argument behaves properly. +func TestIssue31810(t *testing.T) { + // A simple value with no arguments is fine. + var b bytes.Buffer + const text = "{{ (.) }}" + tmpl, err := New("").Parse(text) + if err != nil { + t.Error(err) + } + err = tmpl.Execute(&b, "result") + if err != nil { + t.Error(err) + } + if b.String() != "result" { + t.Errorf("%s got %q, expected %q", text, b.String(), "result") + } + + // Even a plain function fails - need to use call. + f := func() string { return "result" } + b.Reset() + err = tmpl.Execute(&b, f) + if err == nil { + t.Error("expected error with no call, got none") + } + + // Works if the function is explicitly called. + const textCall = "{{ (call .) }}" + tmpl, err = New("").Parse(textCall) + b.Reset() + err = tmpl.Execute(&b, f) + if err != nil { + t.Error(err) + } + if b.String() != "result" { + t.Errorf("%s got %q, expected %q", textCall, b.String(), "result") + } +} diff --git a/tpl/internal/go_templates/texttemplate/funcs.go b/tpl/internal/go_templates/texttemplate/funcs.go index 248dbcf22..1b6940a84 100644 --- a/tpl/internal/go_templates/texttemplate/funcs.go +++ b/tpl/internal/go_templates/texttemplate/funcs.go @@ -12,6 +12,7 @@ import ( "net/url" "reflect" "strings" + "sync" "unicode" "unicode/utf8" ) @@ -29,31 +30,49 @@ import ( // type can return interface{} or reflect.Value. type FuncMap map[string]interface{} -var builtins = FuncMap{ - "and": and, - "call": call, - "html": HTMLEscaper, - "index": index, - "slice": slice, - "js": JSEscaper, - "len": length, - "not": not, - "or": or, - "print": fmt.Sprint, - "printf": fmt.Sprintf, - "println": fmt.Sprintln, - "urlquery": URLQueryEscaper, - - // Comparisons - "eq": eq, // == - "ge": ge, // >= - "gt": gt, // > - "le": le, // <= - "lt": lt, // < - "ne": ne, // != -} - -var builtinFuncs = createValueFuncs(builtins) +// builtins returns the FuncMap. +// It is not a global variable so the linker can dead code eliminate +// more when this isn't called. See golang.org/issue/36021. +// TODO: revert this back to a global map once golang.org/issue/2559 is fixed. +func builtins() FuncMap { + return FuncMap{ + "and": and, + "call": call, + "html": HTMLEscaper, + "index": index, + "slice": slice, + "js": JSEscaper, + "len": length, + "not": not, + "or": or, + "print": fmt.Sprint, + "printf": fmt.Sprintf, + "println": fmt.Sprintln, + "urlquery": URLQueryEscaper, + + // Comparisons + "eq": eq, // == + "ge": ge, // >= + "gt": gt, // > + "le": le, // <= + "lt": lt, // < + "ne": ne, // != + } +} + +var builtinFuncsOnce struct { + sync.Once + v map[string]reflect.Value +} + +// builtinFuncsOnce lazily computes & caches the builtinFuncs map. +// TODO: revert this back to a global map once golang.org/issue/2559 is fixed. +func builtinFuncs() map[string]reflect.Value { + builtinFuncsOnce.Do(func() { + builtinFuncsOnce.v = createValueFuncs(builtins()) + }) + return builtinFuncsOnce.v +} // createValueFuncs turns a FuncMap into a map[string]reflect.Value func createValueFuncs(funcMap FuncMap) map[string]reflect.Value { @@ -125,7 +144,7 @@ func findFunction(name string, tmpl *Template) (reflect.Value, bool) { return fn, true } } - if fn := builtinFuncs[name]; fn.IsValid() { + if fn := builtinFuncs()[name]; fn.IsValid() { return fn, true } return reflect.Value{}, false @@ -185,41 +204,41 @@ func indexArg(index reflect.Value, cap int) (int, error) { // arguments. Thus "index x 1 2 3" is, in Go syntax, x[1][2][3]. Each // indexed item must be a map, slice, or array. func index(item reflect.Value, indexes ...reflect.Value) (reflect.Value, error) { - v := indirectInterface(item) - if !v.IsValid() { + item = indirectInterface(item) + if !item.IsValid() { return reflect.Value{}, fmt.Errorf("index of untyped nil") } - for _, i := range indexes { - index := indirectInterface(i) + for _, index := range indexes { + index = indirectInterface(index) var isNil bool - if v, isNil = indirect(v); isNil { + if item, isNil = indirect(item); isNil { return reflect.Value{}, fmt.Errorf("index of nil pointer") } - switch v.Kind() { + switch item.Kind() { case reflect.Array, reflect.Slice, reflect.String: - x, err := indexArg(index, v.Len()) + x, err := indexArg(index, item.Len()) if err != nil { return reflect.Value{}, err } - v = v.Index(x) + item = item.Index(x) case reflect.Map: - index, err := prepareArg(index, v.Type().Key()) + index, err := prepareArg(index, item.Type().Key()) if err != nil { return reflect.Value{}, err } - if x := v.MapIndex(index); x.IsValid() { - v = x + if x := item.MapIndex(index); x.IsValid() { + item = x } else { - v = reflect.Zero(v.Type().Elem()) + item = reflect.Zero(item.Type().Elem()) } case reflect.Invalid: - // the loop holds invariant: v.IsValid() + // the loop holds invariant: item.IsValid() panic("unreachable") default: - return reflect.Value{}, fmt.Errorf("can't index item of type %s", v.Type()) + return reflect.Value{}, fmt.Errorf("can't index item of type %s", item.Type()) } } - return v, nil + return item, nil } // Slicing. @@ -229,29 +248,27 @@ func index(item reflect.Value, indexes ...reflect.Value) (reflect.Value, error) // is x[:], "slice x 1" is x[1:], and "slice x 1 2 3" is x[1:2:3]. The first // argument must be a string, slice, or array. func slice(item reflect.Value, indexes ...reflect.Value) (reflect.Value, error) { - var ( - cap int - v = indirectInterface(item) - ) - if !v.IsValid() { + item = indirectInterface(item) + if !item.IsValid() { return reflect.Value{}, fmt.Errorf("slice of untyped nil") } if len(indexes) > 3 { return reflect.Value{}, fmt.Errorf("too many slice indexes: %d", len(indexes)) } - switch v.Kind() { + var cap int + switch item.Kind() { case reflect.String: if len(indexes) == 3 { return reflect.Value{}, fmt.Errorf("cannot 3-index slice a string") } - cap = v.Len() + cap = item.Len() case reflect.Array, reflect.Slice: - cap = v.Cap() + cap = item.Cap() default: - return reflect.Value{}, fmt.Errorf("can't slice item of type %s", v.Type()) + return reflect.Value{}, fmt.Errorf("can't slice item of type %s", item.Type()) } // set default values for cases item[:], item[i:]. - idx := [3]int{0, v.Len()} + idx := [3]int{0, item.Len()} for i, index := range indexes { x, err := indexArg(index, cap) if err != nil { @@ -276,20 +293,16 @@ func slice(item reflect.Value, indexes ...reflect.Value) (reflect.Value, error) // Length // length returns the length of the item, with an error if it has no defined length. -func length(item interface{}) (int, error) { - v := reflect.ValueOf(item) - if !v.IsValid() { - return 0, fmt.Errorf("len of untyped nil") - } - v, isNil := indirect(v) +func length(item reflect.Value) (int, error) { + item, isNil := indirect(item) if isNil { return 0, fmt.Errorf("len of nil pointer") } - switch v.Kind() { + switch item.Kind() { case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice, reflect.String: - return v.Len(), nil + return item.Len(), nil } - return 0, fmt.Errorf("len of type %s", v.Type()) + return 0, fmt.Errorf("len of type %s", item.Type()) } // Function invocation @@ -297,11 +310,11 @@ func length(item interface{}) (int, error) { // call returns the result of evaluating the first argument as a function. // The function must return 1 result, or 2 results, the second of which is an error. func call(fn reflect.Value, args ...reflect.Value) (reflect.Value, error) { - v := indirectInterface(fn) - if !v.IsValid() { + fn = indirectInterface(fn) + if !fn.IsValid() { return reflect.Value{}, fmt.Errorf("call of nil") } - typ := v.Type() + typ := fn.Type() if typ.Kind() != reflect.Func { return reflect.Value{}, fmt.Errorf("non-function of type %s", typ) } @@ -322,7 +335,7 @@ func call(fn reflect.Value, args ...reflect.Value) (reflect.Value, error) { } argv := make([]reflect.Value, len(args)) for i, arg := range args { - value := indirectInterface(arg) + arg = indirectInterface(arg) // Compute the expected type. Clumsy because of variadics. argType := dddType if !typ.IsVariadic() || i < numIn-1 { @@ -330,11 +343,11 @@ func call(fn reflect.Value, args ...reflect.Value) (reflect.Value, error) { } var err error - if argv[i], err = prepareArg(value, argType); err != nil { + if argv[i], err = prepareArg(arg, argType); err != nil { return reflect.Value{}, fmt.Errorf("arg %d: %s", i, err) } } - return safeCall(v, argv) + return safeCall(fn, argv) } // safeCall runs fun.Call(args), and returns the resulting value and error, if @@ -440,47 +453,53 @@ func basicKind(v reflect.Value) (kind, error) { // eq evaluates the comparison a == b || a == c || ... func eq(arg1 reflect.Value, arg2 ...reflect.Value) (bool, error) { - v1 := indirectInterface(arg1) - k1, err := basicKind(v1) - if err != nil { - return false, err + arg1 = indirectInterface(arg1) + if arg1 != zero { + if t1 := arg1.Type(); !t1.Comparable() { + return false, fmt.Errorf("uncomparable type %s: %v", t1, arg1) + } } if len(arg2) == 0 { return false, errNoComparison } + k1, _ := basicKind(arg1) for _, arg := range arg2 { - v2 := indirectInterface(arg) - k2, err := basicKind(v2) - if err != nil { - return false, err - } + arg = indirectInterface(arg) + k2, _ := basicKind(arg) truth := false if k1 != k2 { // Special case: Can compare integer values regardless of type's sign. switch { case k1 == intKind && k2 == uintKind: - truth = v1.Int() >= 0 && uint64(v1.Int()) == v2.Uint() + truth = arg1.Int() >= 0 && uint64(arg1.Int()) == arg.Uint() case k1 == uintKind && k2 == intKind: - truth = v2.Int() >= 0 && v1.Uint() == uint64(v2.Int()) + truth = arg.Int() >= 0 && arg1.Uint() == uint64(arg.Int()) default: return false, errBadComparison } } else { switch k1 { case boolKind: - truth = v1.Bool() == v2.Bool() + truth = arg1.Bool() == arg.Bool() case complexKind: - truth = v1.Complex() == v2.Complex() + truth = arg1.Complex() == arg.Complex() case floatKind: - truth = v1.Float() == v2.Float() + truth = arg1.Float() == arg.Float() case intKind: - truth = v1.Int() == v2.Int() + truth = arg1.Int() == arg.Int() case stringKind: - truth = v1.String() == v2.String() + truth = arg1.String() == arg.String() case uintKind: - truth = v1.Uint() == v2.Uint() + truth = arg1.Uint() == arg.Uint() default: - panic("invalid kind") + if arg == zero { + truth = arg1 == arg + } else { + if t2 := arg.Type(); !t2.Comparable() { + return false, fmt.Errorf("uncomparable type %s: %v", t2, arg) + } + truth = arg1.Interface() == arg.Interface() + } } } if truth { @@ -499,13 +518,13 @@ func ne(arg1, arg2 reflect.Value) (bool, error) { // lt evaluates the comparison a < b. func lt(arg1, arg2 reflect.Value) (bool, error) { - v1 := indirectInterface(arg1) - k1, err := basicKind(v1) + arg1 = indirectInterface(arg1) + k1, err := basicKind(arg1) if err != nil { return false, err } - v2 := indirectInterface(arg2) - k2, err := basicKind(v2) + arg2 = indirectInterface(arg2) + k2, err := basicKind(arg2) if err != nil { return false, err } @@ -514,9 +533,9 @@ func lt(arg1, arg2 reflect.Value) (bool, error) { // Special case: Can compare integer values regardless of type's sign. switch { case k1 == intKind && k2 == uintKind: - truth = v1.Int() < 0 || uint64(v1.Int()) < v2.Uint() + truth = arg1.Int() < 0 || uint64(arg1.Int()) < arg2.Uint() case k1 == uintKind && k2 == intKind: - truth = v2.Int() >= 0 && v1.Uint() < uint64(v2.Int()) + truth = arg2.Int() >= 0 && arg1.Uint() < uint64(arg2.Int()) default: return false, errBadComparison } @@ -525,13 +544,13 @@ func lt(arg1, arg2 reflect.Value) (bool, error) { case boolKind, complexKind: return false, errBadComparisonType case floatKind: - truth = v1.Float() < v2.Float() + truth = arg1.Float() < arg2.Float() case intKind: - truth = v1.Int() < v2.Int() + truth = arg1.Int() < arg2.Int() case stringKind: - truth = v1.String() < v2.String() + truth = arg1.String() < arg2.String() case uintKind: - truth = v1.Uint() < v2.Uint() + truth = arg1.Uint() < arg2.Uint() default: panic("invalid kind") } @@ -634,8 +653,10 @@ var ( jsBackslash = []byte(`\\`) jsApos = []byte(`\'`) jsQuot = []byte(`\"`) - jsLt = []byte(`\x3C`) - jsGt = []byte(`\x3E`) + jsLt = []byte(`\u003C`) + jsGt = []byte(`\u003E`) + jsAmp = []byte(`\u0026`) + jsEq = []byte(`\u003D`) ) // JSEscape writes to w the escaped JavaScript equivalent of the plain text data b. @@ -664,6 +685,10 @@ func JSEscape(w io.Writer, b []byte) { w.Write(jsLt) case '>': w.Write(jsGt) + case '&': + w.Write(jsAmp) + case '=': + w.Write(jsEq) default: w.Write(jsLowUni) t, b := c>>4, c&0x0f @@ -698,7 +723,7 @@ func JSEscapeString(s string) string { func jsIsSpecial(r rune) bool { switch r { - case '\\', '\'', '"', '<', '>': + case '\\', '\'', '"', '<', '>', '&', '=': return true } return r < ' ' || utf8.RuneSelf <= r diff --git a/tpl/internal/go_templates/texttemplate/hugo_template.go b/tpl/internal/go_templates/texttemplate/hugo_template.go index 37fa969da..7cd6df0fb 100644 --- a/tpl/internal/go_templates/texttemplate/hugo_template.go +++ b/tpl/internal/go_templates/texttemplate/hugo_template.go @@ -30,7 +30,7 @@ package is auto generated. */ // Export it so we can populate Hugo's func map with it, which makes it faster. -var GoFuncs = builtinFuncs +var GoFuncs = builtinFuncs() // Preparer prepares the template before execution. type Preparer interface { diff --git a/tpl/internal/go_templates/texttemplate/multi_test.go b/tpl/internal/go_templates/texttemplate/multi_test.go index e41bb9d8e..7323be379 100644 --- a/tpl/internal/go_templates/texttemplate/multi_test.go +++ b/tpl/internal/go_templates/texttemplate/multi_test.go @@ -244,7 +244,7 @@ func TestAddParseTree(t *testing.T) { t.Fatal(err) } // Add a new parse tree. - tree, err := parse.Parse("cloneText3", cloneText3, "", "", nil, builtins) + tree, err := parse.Parse("cloneText3", cloneText3, "", "", nil, builtins()) if err != nil { t.Fatal(err) } diff --git a/tpl/internal/go_templates/texttemplate/parse/lex.go b/tpl/internal/go_templates/texttemplate/parse/lex.go index 3d5770879..30371f286 100644 --- a/tpl/internal/go_templates/texttemplate/parse/lex.go +++ b/tpl/internal/go_templates/texttemplate/parse/lex.go @@ -411,7 +411,6 @@ func lexInsideAction(l *lexer) stateFn { } case r <= unicode.MaxASCII && unicode.IsPrint(r): l.emit(itemChar) - return lexInsideAction default: return l.errorf("unrecognized character in action: %#U", r) } diff --git a/tpl/internal/go_templates/texttemplate/parse/node.go b/tpl/internal/go_templates/texttemplate/parse/node.go index 1174a4b97..1c116ea6f 100644 --- a/tpl/internal/go_templates/texttemplate/parse/node.go +++ b/tpl/internal/go_templates/texttemplate/parse/node.go @@ -7,7 +7,6 @@ package parse import ( - "bytes" "fmt" "strconv" "strings" @@ -29,6 +28,8 @@ type Node interface { // tree returns the containing *Tree. // It is unexported so all implementations of Node are in this package. tree() *Tree + // writeTo writes the String output to the builder. + writeTo(*strings.Builder) } // NodeType identifies the type of a parse tree node. @@ -94,11 +95,15 @@ func (l *ListNode) tree() *Tree { } func (l *ListNode) String() string { - b := new(bytes.Buffer) + var sb strings.Builder + l.writeTo(&sb) + return sb.String() +} + +func (l *ListNode) writeTo(sb *strings.Builder) { for _, n := range l.Nodes { - fmt.Fprint(b, n) + n.writeTo(sb) } - return b.String() } func (l *ListNode) CopyList() *ListNode { @@ -132,6 +137,10 @@ func (t *TextNode) String() string { return fmt.Sprintf(textFormat, t.Text) } +func (t *TextNode) writeTo(sb *strings.Builder) { + sb.WriteString(t.String()) +} + func (t *TextNode) tree() *Tree { return t.tr } @@ -160,23 +169,27 @@ func (p *PipeNode) append(command *CommandNode) { } func (p *PipeNode) String() string { - s := "" + var sb strings.Builder + p.writeTo(&sb) + return sb.String() +} + +func (p *PipeNode) writeTo(sb *strings.Builder) { if len(p.Decl) > 0 { for i, v := range p.Decl { if i > 0 { - s += ", " + sb.WriteString(", ") } - s += v.String() + v.writeTo(sb) } - s += " := " + sb.WriteString(" := ") } for i, c := range p.Cmds { if i > 0 { - s += " | " + sb.WriteString(" | ") } - s += c.String() + c.writeTo(sb) } - return s } func (p *PipeNode) tree() *Tree { @@ -187,9 +200,9 @@ func (p *PipeNode) CopyPipe() *PipeNode { if p == nil { return p } - var vars []*VariableNode - for _, d := range p.Decl { - vars = append(vars, d.Copy().(*VariableNode)) + vars := make([]*VariableNode, len(p.Decl)) + for i, d := range p.Decl { + vars[i] = d.Copy().(*VariableNode) } n := p.tr.newPipeline(p.Pos, p.Line, vars) n.IsAssign = p.IsAssign @@ -219,8 +232,15 @@ func (t *Tree) newAction(pos Pos, line int, pipe *PipeNode) *ActionNode { } func (a *ActionNode) String() string { - return fmt.Sprintf("{{%s}}", a.Pipe) + var sb strings.Builder + a.writeTo(&sb) + return sb.String() +} +func (a *ActionNode) writeTo(sb *strings.Builder) { + sb.WriteString("{{") + a.Pipe.writeTo(sb) + sb.WriteString("}}") } func (a *ActionNode) tree() *Tree { @@ -249,18 +269,24 @@ func (c *CommandNode) append(arg Node) { } func (c *CommandNode) String() string { - s := "" + var sb strings.Builder + c.writeTo(&sb) + return sb.String() +} + +func (c *CommandNode) writeTo(sb *strings.Builder) { for i, arg := range c.Args { if i > 0 { - s += " " + sb.WriteByte(' ') } if arg, ok := arg.(*PipeNode); ok { - s += "(" + arg.String() + ")" + sb.WriteByte('(') + arg.writeTo(sb) + sb.WriteByte(')') continue } - s += arg.String() + arg.writeTo(sb) } - return s } func (c *CommandNode) tree() *Tree { @@ -311,6 +337,10 @@ func (i *IdentifierNode) String() string { return i.Ident } +func (i *IdentifierNode) writeTo(sb *strings.Builder) { + sb.WriteString(i.String()) +} + func (i *IdentifierNode) tree() *Tree { return i.tr } @@ -333,14 +363,18 @@ func (t *Tree) newVariable(pos Pos, ident string) *VariableNode { } func (v *VariableNode) String() string { - s := "" + var sb strings.Builder + v.writeTo(&sb) + return sb.String() +} + +func (v *VariableNode) writeTo(sb *strings.Builder) { for i, id := range v.Ident { if i > 0 { - s += "." + sb.WriteByte('.') } - s += id + sb.WriteString(id) } - return s } func (v *VariableNode) tree() *Tree { @@ -373,6 +407,10 @@ func (d *DotNode) String() string { return "." } +func (d *DotNode) writeTo(sb *strings.Builder) { + sb.WriteString(d.String()) +} + func (d *DotNode) tree() *Tree { return d.tr } @@ -403,6 +441,10 @@ func (n *NilNode) String() string { return "nil" } +func (n *NilNode) writeTo(sb *strings.Builder) { + sb.WriteString(n.String()) +} + func (n *NilNode) tree() *Tree { return n.tr } @@ -426,11 +468,16 @@ func (t *Tree) newField(pos Pos, ident string) *FieldNode { } func (f *FieldNode) String() string { - s := "" + var sb strings.Builder + f.writeTo(&sb) + return sb.String() +} + +func (f *FieldNode) writeTo(sb *strings.Builder) { for _, id := range f.Ident { - s += "." + id + sb.WriteByte('.') + sb.WriteString(id) } - return s } func (f *FieldNode) tree() *Tree { @@ -469,14 +516,23 @@ func (c *ChainNode) Add(field string) { } func (c *ChainNode) String() string { - s := c.Node.String() + var sb strings.Builder + c.writeTo(&sb) + return sb.String() +} + +func (c *ChainNode) writeTo(sb *strings.Builder) { if _, ok := c.Node.(*PipeNode); ok { - s = "(" + s + ")" + sb.WriteByte('(') + c.Node.writeTo(sb) + sb.WriteByte(')') + } else { + c.Node.writeTo(sb) } for _, field := range c.Field { - s += "." + field + sb.WriteByte('.') + sb.WriteString(field) } - return s } func (c *ChainNode) tree() *Tree { @@ -506,6 +562,10 @@ func (b *BoolNode) String() string { return "false" } +func (b *BoolNode) writeTo(sb *strings.Builder) { + sb.WriteString(b.String()) +} + func (b *BoolNode) tree() *Tree { return b.tr } @@ -639,6 +699,10 @@ func (n *NumberNode) String() string { return n.Text } +func (n *NumberNode) writeTo(sb *strings.Builder) { + sb.WriteString(n.String()) +} + func (n *NumberNode) tree() *Tree { return n.tr } @@ -666,6 +730,10 @@ func (s *StringNode) String() string { return s.Quoted } +func (s *StringNode) writeTo(sb *strings.Builder) { + sb.WriteString(s.String()) +} + func (s *StringNode) tree() *Tree { return s.tr } @@ -690,6 +758,10 @@ func (e *endNode) String() string { return "{{end}}" } +func (e *endNode) writeTo(sb *strings.Builder) { + sb.WriteString(e.String()) +} + func (e *endNode) tree() *Tree { return e.tr } @@ -718,6 +790,10 @@ func (e *elseNode) String() string { return "{{else}}" } +func (e *elseNode) writeTo(sb *strings.Builder) { + sb.WriteString(e.String()) +} + func (e *elseNode) tree() *Tree { return e.tr } @@ -738,6 +814,12 @@ type BranchNode struct { } func (b *BranchNode) String() string { + var sb strings.Builder + b.writeTo(&sb) + return sb.String() +} + +func (b *BranchNode) writeTo(sb *strings.Builder) { name := "" switch b.NodeType { case NodeIf: @@ -749,10 +831,17 @@ func (b *BranchNode) String() string { default: panic("unknown branch type") } + sb.WriteString("{{") + sb.WriteString(name) + sb.WriteByte(' ') + b.Pipe.writeTo(sb) + sb.WriteString("}}") + b.List.writeTo(sb) if b.ElseList != nil { - return fmt.Sprintf("{{%s %s}}%s{{else}}%s{{end}}", name, b.Pipe, b.List, b.ElseList) + sb.WriteString("{{else}}") + b.ElseList.writeTo(sb) } - return fmt.Sprintf("{{%s %s}}%s{{end}}", name, b.Pipe, b.List) + sb.WriteString("{{end}}") } func (b *BranchNode) tree() *Tree { @@ -826,10 +915,19 @@ func (t *Tree) newTemplate(pos Pos, line int, name string, pipe *PipeNode) *Temp } func (t *TemplateNode) String() string { - if t.Pipe == nil { - return fmt.Sprintf("{{template %q}}", t.Name) + var sb strings.Builder + t.writeTo(&sb) + return sb.String() +} + +func (t *TemplateNode) writeTo(sb *strings.Builder) { + sb.WriteString("{{template ") + sb.WriteString(strconv.Quote(t.Name)) + if t.Pipe != nil { + sb.WriteByte(' ') + t.Pipe.writeTo(sb) } - return fmt.Sprintf("{{template %q %s}}", t.Name, t.Pipe) + sb.WriteString("}}") } func (t *TemplateNode) tree() *Tree { diff --git a/tpl/internal/go_templates/texttemplate/parse/parse.go b/tpl/internal/go_templates/texttemplate/parse/parse.go index 7c35b0ff3..c9b80f4a2 100644 --- a/tpl/internal/go_templates/texttemplate/parse/parse.go +++ b/tpl/internal/go_templates/texttemplate/parse/parse.go @@ -108,13 +108,8 @@ func (t *Tree) nextNonSpace() (token item) { } // peekNonSpace returns but does not consume the next non-space token. -func (t *Tree) peekNonSpace() (token item) { - for { - token = t.next() - if token.typ != itemSpace { - break - } - } +func (t *Tree) peekNonSpace() item { + token := t.nextNonSpace() t.backup() return token } diff --git a/tpl/internal/go_templates/texttemplate/parse/parse_test.go b/tpl/internal/go_templates/texttemplate/parse/parse_test.go index 5d3b59b63..79e7bb5ae 100644 --- a/tpl/internal/go_templates/texttemplate/parse/parse_test.go +++ b/tpl/internal/go_templates/texttemplate/parse/parse_test.go @@ -306,7 +306,8 @@ var parseTests = []parseTest{ } var builtins = map[string]interface{}{ - "printf": fmt.Sprintf, + "printf": fmt.Sprintf, + "contains": strings.Contains, } func testParse(doCopy bool, t *testing.T) { @@ -555,3 +556,52 @@ func BenchmarkParseLarge(b *testing.B) { } } } + +var sinkv, sinkl string + +func BenchmarkVariableString(b *testing.B) { + v := &VariableNode{ + Ident: []string{"$", "A", "BB", "CCC", "THIS_IS_THE_VARIABLE_BEING_PROCESSED"}, + } + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + sinkv = v.String() + } + if sinkv == "" { + b.Fatal("Benchmark was not run") + } +} + +func BenchmarkListString(b *testing.B) { + text := ` +{{(printf .Field1.Field2.Field3).Value}} +{{$x := (printf .Field1.Field2.Field3).Value}} +{{$y := (printf $x.Field1.Field2.Field3).Value}} +{{$z := $y.Field1.Field2.Field3}} +{{if contains $y $z}} + {{printf "%q" $y}} +{{else}} + {{printf "%q" $x}} +{{end}} +{{with $z.Field1 | contains "boring"}} + {{printf "%q" . | printf "%s"}} +{{else}} + {{printf "%d %d %d" 11 11 11}} + {{printf "%d %d %s" 22 22 $x.Field1.Field2.Field3 | printf "%s"}} + {{printf "%v" (contains $z.Field1.Field2 $y)}} +{{end}} +` + tree, err := New("bench").Parse(text, "", "", make(map[string]*Tree), builtins) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + sinkl = tree.Root.String() + } + if sinkl == "" { + b.Fatal("Benchmark was not run") + } +} diff --git a/tpl/internal/go_templates/texttemplate/template.go b/tpl/internal/go_templates/texttemplate/template.go index 4df8a9ed0..9c6ba6dfc 100644 --- a/tpl/internal/go_templates/texttemplate/template.go +++ b/tpl/internal/go_templates/texttemplate/template.go @@ -110,20 +110,21 @@ func (t *Template) Clone() (*Template, error) { // copy returns a shallow copy of t, with common set to the argument. func (t *Template) copy(c *common) *Template { - nt := New(t.name) - nt.Tree = t.Tree - nt.common = c - nt.leftDelim = t.leftDelim - nt.rightDelim = t.rightDelim - return nt + return &Template{ + name: t.name, + Tree: t.Tree, + common: c, + leftDelim: t.leftDelim, + rightDelim: t.rightDelim, + } } -// AddParseTree adds parse tree for template with given name and associates it with t. -// If the template does not already exist, it will create a new one. -// If the template does exist, it will be replaced. +// AddParseTree associates the argument parse tree with the template t, giving +// it the specified name. If the template has not been defined, this tree becomes +// its definition. If it has been defined and already has that name, the existing +// definition is replaced; otherwise a new template is created, defined, and returned. func (t *Template) AddParseTree(name string, tree *parse.Tree) (*Template, error) { t.init() - // If the name is the name of this template, overwrite this template. nt := t if name != t.name { nt = t.New(name) @@ -197,7 +198,7 @@ func (t *Template) Lookup(name string) *Template { func (t *Template) Parse(text string) (*Template, error) { t.init() t.muFuncs.RLock() - trees, err := parse.Parse(t.name, text, t.leftDelim, t.rightDelim, t.parseFuncs, builtins) + trees, err := parse.Parse(t.name, text, t.leftDelim, t.rightDelim, t.parseFuncs, builtins()) t.muFuncs.RUnlock() if err != nil { return nil, err |