diff options
| author | Bjørn Erik Pedersen <[email protected]> | 2022-11-14 19:13:09 +0100 |
|---|---|---|
| committer | Bjørn Erik Pedersen <[email protected]> | 2022-11-14 22:31:50 +0100 |
| commit | f6ab9553f4c0429586fc9221d1779c460cf4922a (patch) | |
| tree | 0f4efed30fb9750b800a4865c5065285bbc4d1fc | |
| parent | 58a98c7758f90a16df51e4fee9ead0233157a1e4 (diff) | |
| download | hugo-f6ab9553f4c0429586fc9221d1779c460cf4922a.tar.gz hugo-f6ab9553f4c0429586fc9221d1779c460cf4922a.zip | |
tpl/internal: Sync go_templates
Closes #10411
34 files changed, 740 insertions, 515 deletions
diff --git a/scripts/fork_go_templates/main.go b/scripts/fork_go_templates/main.go index 9296b7bdd..2f8516e8d 100644 --- a/scripts/fork_go_templates/main.go +++ b/scripts/fork_go_templates/main.go @@ -17,7 +17,7 @@ import ( ) func main() { - // The current is built with 41a82aa9c3 text/template/parse: allow space after continue or break + // The current is built with be7068fb0804f661515c678bee9224b90b32869a text/template: correct assignment, not declaration, in range fmt.Println("Forking ...") defer fmt.Println("Done ...") diff --git a/tpl/internal/go_templates/fmtsort/sort.go b/tpl/internal/go_templates/fmtsort/sort.go index 34c1f477f..278a89bd7 100644 --- a/tpl/internal/go_templates/fmtsort/sort.go +++ b/tpl/internal/go_templates/fmtsort/sort.go @@ -36,19 +36,18 @@ func (o *SortedMap) Swap(i, j int) { // // The ordering rules are more general than with Go's < operator: // -// - when applicable, nil compares low -// - ints, floats, and strings order by < -// - NaN compares less than non-NaN floats -// - bool compares false before true -// - complex compares real, then imag -// - pointers compare by machine address -// - channel values compare by machine address -// - structs compare each field in turn -// - arrays compare each element in turn. -// Otherwise identical arrays compare by length. -// - interface values compare first by reflect.Type describing the concrete type -// and then by concrete value as described in the previous rules. -// +// - when applicable, nil compares low +// - ints, floats, and strings order by < +// - NaN compares less than non-NaN floats +// - bool compares false before true +// - complex compares real, then imag +// - pointers compare by machine address +// - channel values compare by machine address +// - structs compare each field in turn +// - arrays compare each element in turn. +// Otherwise identical arrays compare by length. +// - interface values compare first by reflect.Type describing the concrete type +// and then by concrete value as described in the previous rules. func Sort(mapValue reflect.Value) *SortedMap { if mapValue.Type().Kind() != reflect.Map { return nil diff --git a/tpl/internal/go_templates/fmtsort/sort_test.go b/tpl/internal/go_templates/fmtsort/sort_test.go index a05e8a3c3..064b09107 100644 --- a/tpl/internal/go_templates/fmtsort/sort_test.go +++ b/tpl/internal/go_templates/fmtsort/sort_test.go @@ -147,7 +147,7 @@ func sprint(data any) string { } b.WriteString(sprintKey(key)) b.WriteRune(':') - b.WriteString(fmt.Sprint(om.Value[i])) + fmt.Fprint(b, om.Value[i]) } return b.String() } diff --git a/tpl/internal/go_templates/htmltemplate/clone_test.go b/tpl/internal/go_templates/htmltemplate/clone_test.go index 553f656b5..7db335b5b 100644 --- a/tpl/internal/go_templates/htmltemplate/clone_test.go +++ b/tpl/internal/go_templates/htmltemplate/clone_test.go @@ -8,7 +8,6 @@ package template import ( - "bytes" "errors" "fmt" "io" @@ -26,7 +25,7 @@ func TestAddParseTreeHTML(t *testing.T) { t.Fatal(err) } added := Must(root.AddParseTree("b", tree["b"])) - b := new(bytes.Buffer) + b := new(strings.Builder) err = added.ExecuteTemplate(b, "a", "1>0") if err != nil { t.Fatal(err) @@ -43,7 +42,7 @@ func TestClone(t *testing.T) { // In the t2 template, it will be in a JavaScript context. // In the t3 template, it will be in a CSS context. const tmpl = `{{define "a"}}{{template "lhs"}}{{.}}{{template "rhs"}}{{end}}` - b := new(bytes.Buffer) + b := new(strings.Builder) // Create an incomplete template t0. t0 := Must(New("t0").Parse(tmpl)) diff --git a/tpl/internal/go_templates/htmltemplate/content_test.go b/tpl/internal/go_templates/htmltemplate/content_test.go index 29221a4ad..fac4774cc 100644 --- a/tpl/internal/go_templates/htmltemplate/content_test.go +++ b/tpl/internal/go_templates/htmltemplate/content_test.go @@ -284,7 +284,7 @@ func TestTypedContent(t *testing.T) { []string{ `#ZgotmplZ`, `#ZgotmplZ`, - // Commas are not esacped + // Commas are not escaped. `Hello,#ZgotmplZ`, // Leading spaces are not percent escapes. ` dir=%22ltr%22`, @@ -389,7 +389,7 @@ func TestTypedContent(t *testing.T) { tmpl := Must(New("x").Parse(test.input)) pre := strings.Index(test.input, "{{.}}") post := len(test.input) - (pre + 5) - var b bytes.Buffer + var b strings.Builder for i, x := range data { b.Reset() if err := tmpl.Execute(&b, x); err != nil { @@ -423,7 +423,7 @@ func (s *errorer) Error() string { func TestStringer(t *testing.T) { s := &myStringer{3} - b := new(bytes.Buffer) + b := new(strings.Builder) tmpl := Must(New("x").Parse("{{.}}")) if err := tmpl.Execute(b, s); err != nil { t.Fatal(err) diff --git a/tpl/internal/go_templates/htmltemplate/context.go b/tpl/internal/go_templates/htmltemplate/context.go index c28e08dce..146a95d03 100644 --- a/tpl/internal/go_templates/htmltemplate/context.go +++ b/tpl/internal/go_templates/htmltemplate/context.go @@ -80,7 +80,9 @@ func (c context) mangle(templateName string) string { // HTML5 parsing algorithm because a single token production in the HTML // grammar may contain embedded actions in a template. For instance, the quoted // HTML attribute produced by -// <div title="Hello {{.World}}"> +// +// <div title="Hello {{.World}}"> +// // is a single token in HTML's grammar but in a template spans several nodes. type state uint8 diff --git a/tpl/internal/go_templates/htmltemplate/doc.go b/tpl/internal/go_templates/htmltemplate/doc.go index b6a1504f8..8422b4921 100644 --- a/tpl/internal/go_templates/htmltemplate/doc.go +++ b/tpl/internal/go_templates/htmltemplate/doc.go @@ -12,14 +12,14 @@ The documentation here focuses on the security features of the package. For information about how to program the templates themselves, see the documentation for text/template. -Introduction +# Introduction This package wraps package text/template so you can share its template API to parse and execute HTML templates safely. - tmpl, err := template.New("name").Parse(...) - // Error checking elided - err = tmpl.Execute(out, data) + tmpl, err := template.New("name").Parse(...) + // Error checking elided + err = tmpl.Execute(out, data) If successful, tmpl will now be injection-safe. Otherwise, err is an error defined in the docs for ErrorCode. @@ -34,38 +34,37 @@ provided below. Example - import template "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate" - ... - t, err := template.New("foo").Parse(`{{define "T"}}Hello, {{.}}!{{end}}`) - err = t.ExecuteTemplate(out, "T", "<script>alert('you have been pwned')</script>") + import template "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate" + ... + t, err := template.New("foo").Parse(`{{define "T"}}Hello, {{.}}!{{end}}`) + err = t.ExecuteTemplate(out, "T", "<script>alert('you have been pwned')</script>") produces - Hello, <script>alert('you have been pwned')</script>! + Hello, <script>alert('you have been pwned')</script>! but the contextual autoescaping in html/template - import template "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate" - ... - t, err := template.New("foo").Parse(`{{define "T"}}Hello, {{.}}!{{end}}`) - err = t.ExecuteTemplate(out, "T", "<script>alert('you have been pwned')</script>") + import template "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate" + ... + t, err := template.New("foo").Parse(`{{define "T"}}Hello, {{.}}!{{end}}`) + err = t.ExecuteTemplate(out, "T", "<script>alert('you have been pwned')</script>") produces safe, escaped HTML output - Hello, <script>alert('you have been pwned')</script>! + Hello, <script>alert('you have been pwned')</script>! - -Contexts +# Contexts This package understands HTML, CSS, JavaScript, and URIs. It adds sanitizing functions to each simple action pipeline, so given the excerpt - <a href="/search?q={{.}}">{{.}}</a> + <a href="/search?q={{.}}">{{.}}</a> At parse time each {{.}} is overwritten to add escaping functions as necessary. In this case it becomes - <a href="/search?q={{. | urlescaper | attrescaper}}">{{. | htmlescaper}}</a> + <a href="/search?q={{. | urlescaper | attrescaper}}">{{. | htmlescaper}}</a> where urlescaper, attrescaper, and htmlescaper are aliases for internal escaping functions. @@ -73,117 +72,113 @@ functions. For these internal escaping functions, if an action pipeline evaluates to a nil interface value, it is treated as though it were an empty string. -Namespaced and data- attributes +# Namespaced and data- attributes Attributes with a namespace are treated as if they had no namespace. Given the excerpt - <a my:href="{{.}}"></a> + <a my:href="{{.}}"></a> At parse time the attribute will be treated as if it were just "href". So at parse time the template becomes: - <a my:href="{{. | urlescaper | attrescaper}}"></a> + <a my:href="{{. | urlescaper | attrescaper}}"></a> Similarly to attributes with namespaces, attributes with a "data-" prefix are treated as if they had no "data-" prefix. So given - <a data-href="{{.}}"></a> + <a data-href="{{.}}"></a> At parse time this becomes - <a data-href="{{. | urlescaper | attrescaper}}"></a> + <a data-href="{{. | urlescaper | attrescaper}}"></a> If an attribute has both a namespace and a "data-" prefix, only the namespace will be removed when determining the context. For example - <a my:data-href="{{.}}"></a> + <a my:data-href="{{.}}"></a> This is handled as if "my:data-href" was just "data-href" and not "href" as it would be if the "data-" prefix were to be ignored too. Thus at parse time this becomes just - <a my:data-href="{{. | attrescaper}}"></a> + <a my:data-href="{{. | attrescaper}}"></a> As a special case, attributes with the namespace "xmlns" are always treated as containing URLs. Given the excerpts - <a xmlns:title="{{.}}"></a> - <a xmlns:href="{{.}}"></a> - <a xmlns:onclick="{{.}}"></a> + <a xmlns:title="{{.}}"></a> + <a xmlns:href="{{.}}"></a> + <a xmlns:onclick="{{.}}"></a> At parse time they become: - <a xmlns:title="{{. | urlescaper | attrescaper}}"></a> - <a xmlns:href="{{. | urlescaper | attrescaper}}"></a> - <a xmlns:onclick="{{. | urlescaper | attrescaper}}"></a> + <a xmlns:title="{{. | urlescaper | attrescaper}}"></a> + <a xmlns:href="{{. | urlescaper | attrescaper}}"></a> + <a xmlns:onclick="{{. | urlescaper | attrescaper}}"></a> -Errors +# Errors See the documentation of ErrorCode for details. - -A fuller picture +# A fuller picture The rest of this package comment may be skipped on first reading; it includes details necessary to understand escaping contexts and error messages. Most users will not need to understand these details. - -Contexts +# Contexts Assuming {{.}} is `O'Reilly: How are <i>you</i>?`, the table below shows how {{.}} appears when used in the context to the left. - Context {{.}} After - {{.}} O'Reilly: How are <i>you</i>? - <a title='{{.}}'> O'Reilly: How are you? - <a href="/{{.}}"> O'Reilly: How are %3ci%3eyou%3c/i%3e? - <a href="?q={{.}}"> O'Reilly%3a%20How%20are%3ci%3e...%3f - <a onx='f("{{.}}")'> O\x27Reilly: How are \x3ci\x3eyou...? - <a onx='f({{.}})'> "O\x27Reilly: How are \x3ci\x3eyou...?" - <a onx='pattern = /{{.}}/;'> O\x27Reilly: How are \x3ci\x3eyou...\x3f + Context {{.}} After + {{.}} O'Reilly: How are <i>you</i>? + <a title='{{.}}'> O'Reilly: How are you? + <a href="/{{.}}"> O'Reilly: How are %3ci%3eyou%3c/i%3e? + <a href="?q={{.}}"> O'Reilly%3a%20How%20are%3ci%3e...%3f + <a onx='f("{{.}}")'> O\x27Reilly: How are \x3ci\x3eyou...? + <a onx='f({{.}})'> "O\x27Reilly: How are \x3ci\x3eyou...?" + <a onx='pattern = /{{.}}/;'> O\x27Reilly: How are \x3ci\x3eyou...\x3f If used in an unsafe context, then the value might be filtered out: - Context {{.}} After - <a href="{{.}}"> #ZgotmplZ + Context {{.}} After + <a href="{{.}}"> #ZgotmplZ since "O'Reilly:" is not an allowed protocol like "http:". - If {{.}} is the innocuous word, `left`, then it can appear more widely, - Context {{.}} After - {{.}} left - <a title='{{.}}'> left - <a href='{{.}}'> left - <a href='/{{.}}'> left - <a href='?dir={{.}}'> left - <a style="border-{{.}}: 4px"> left - <a style="align: {{.}}"> left - <a style="background: '{{.}}'> left - <a style="background: url('{{.}}')> left - <style>p.{{.}} {color:red}</style> left + Context {{.}} After + {{.}} left + <a title='{{.}}'> left + <a href='{{.}}'> left + <a href='/{{.}}'> left + <a href='?dir={{.}}'> left + <a style="border-{{.}}: 4px"> left + <a style="align: {{.}}"> left + <a style="background: '{{.}}'> left + <a style="background: url('{{.}}')> left + <style>p.{{.}} {color:red}</style> left Non-string values can be used in JavaScript contexts. If {{.}} is - struct{A,B string}{ "foo", "bar" } + struct{A,B string}{ "foo", "bar" } in the escaped template - <script>var pair = {{.}};</script> + <script>var pair = {{.}};</script> then the template output is - <script>var pair = {"A": "foo", "B": "bar"};</script> + <script>var pair = {"A": "foo", "B": "bar"};</script> See package json to understand how non-string content is marshaled for embedding in JavaScript contexts. - -Typed Strings +# Typed Strings By default, this package assumes that all pipelines produce a plain text string. It adds escaping pipeline stages necessary to correctly and safely embed that @@ -197,24 +192,23 @@ exempted from escaping. The template - Hello, {{.}}! + Hello, {{.}}! can be invoked with - tmpl.Execute(out, template.HTML(`<b>World</b>`)) + tmpl.Execute(out, template.HTML(`<b>World</b>`)) to produce - Hello, <b>World</b>! + Hello, <b>World</b>! instead of the - Hello, <b>World<b>! + Hello, <b>World<b>! that would have been produced if {{.}} was a regular string. - -Security Model +# Security Model https://rawgit.com/mikesamuel/sanitized-jquery-templates/trunk/safetemplate.html#problem_definition defines "safe" as used by this package. diff --git a/tpl/internal/go_templates/htmltemplate/error.go b/tpl/internal/go_templates/htmltemplate/error.go index 21c86a9ef..916b41a82 100644 --- a/tpl/internal/go_templates/htmltemplate/error.go +++ b/tpl/internal/go_templates/htmltemplate/error.go @@ -33,14 +33,17 @@ type ErrorCode int // // Output: "ZgotmplZ" // Example: -// <img src="{{.X}}"> -// where {{.X}} evaluates to `javascript:...` +// +// <img src="{{.X}}"> +// where {{.X}} evaluates to `javascript:...` +// // Discussion: -// "ZgotmplZ" is a special value that indicates that unsafe content reached a -// CSS or URL context at runtime. The output of the example will be -// <img src="#ZgotmplZ"> -// If the data comes from a trusted source, use content types to exempt it -// from filtering: URL(`javascript:...`). +// +// "ZgotmplZ" is a special value that indicates that unsafe content reached a +// CSS or URL context at runtime. The output of the example will be +// <img src="#ZgotmplZ"> +// If the data comes from a trusted source, use content types to exempt it +// from filtering: URL(`javascript:...`). const ( // OK indicates the lack of an error. OK ErrorCode = iota diff --git a/tpl/internal/go_templates/htmltemplate/escape.go b/tpl/internal/go_templates/htmltemplate/escape.go index 488894416..3aac865ef 100644 --- a/tpl/internal/go_templates/htmltemplate/escape.go +++ b/tpl/internal/go_templates/htmltemplate/escape.go @@ -412,13 +412,19 @@ func newIdentCmd(identifier string, pos parse.Pos) *parse.CommandNode { // nudge returns the context that would result from following empty string // transitions from the input context. // For example, parsing: -// `<a href=` +// +// `<a href=` +// // will end in context{stateBeforeValue, attrURL}, but parsing one extra rune: -// `<a href=x` +// +// `<a href=x` +// // will end in context{stateURL, delimSpaceOrTagEnd, ...}. // There are two transitions that happen when the 'x' is seen: // (1) Transition from a before-value state to a start-of-value state without -// consuming any character. +// +// consuming any character. +// // (2) Consume 'x' and transition past the first value character. // In this case, nudging produces the context after (1) happens. func nudge(c context) context { diff --git a/tpl/internal/go_templates/htmltemplate/escape_test.go b/tpl/internal/go_templates/htmltemplate/escape_test.go index adf160b5d..a08ea57ef 100644 --- a/tpl/internal/go_templates/htmltemplate/escape_test.go +++ b/tpl/internal/go_templates/htmltemplate/escape_test.go @@ -693,7 +693,7 @@ func TestEscape(t *testing.T) { t.Errorf("%s: tree not set properly", test.name) continue } - b := new(bytes.Buffer) + b := new(strings.Builder) if err := tmpl.Execute(b, data); err != nil { t.Errorf("%s: template execution failed: %s", test.name, err) continue @@ -740,7 +740,7 @@ func TestEscapeMap(t *testing.T) { }, } { tmpl := Must(New("").Parse(test.input)) - b := new(bytes.Buffer) + b := new(strings.Builder) if err := tmpl.Execute(b, data); err != nil { t.Errorf("%s: template execution failed: %s", test.desc, err) continue @@ -882,7 +882,7 @@ func TestEscapeSet(t *testing.T) { t.Errorf("error parsing %q: %v", source, err) continue } - var b bytes.Buffer + var b strings.Builder if err := tmpl.ExecuteTemplate(&b, "main", data); err != nil { t.Errorf("%q executing %v", err.Error(), tmpl.Lookup("main")) @@ -1833,7 +1833,7 @@ func TestIndirectPrint(t *testing.T) { bp := &b bpp := &bp tmpl := Must(New("t").Parse(`{{.}}`)) - var buf bytes.Buffer + var buf strings.Builder err := tmpl.Execute(&buf, ap) if err != nil { t.Errorf("Unexpected error: %s", err) @@ -1876,7 +1876,7 @@ func TestPipeToMethodIsEscaped(t *testing.T) { t.Errorf("panicked: %v\n", panicValue) } }() - var b bytes.Buffer + var b strings.Builder tmpl.Execute(&b, Issue7379(0)) return b.String() } @@ -1909,7 +1909,7 @@ func TestIdempotentExecute(t *testing.T) { Parse(`{{define "main"}}<body>{{template "hello"}}</body>{{end}}`)) Must(tmpl. Parse(`{{define "hello"}}Hello, {{"Ladies & Gentlemen!"}}{{end}}`)) - got := new(bytes.Buffer) + got := new(strings.Builder) var err error // Ensure that "hello" produces the same output when executed twice. want := "Hello, Ladies & Gentlemen!" @@ -1952,7 +1952,7 @@ func TestOrphanedTemplate(t *testing.T) { t1 := Must(New("foo").Parse(`<a href="{{.}}">link1</a>`)) t2 := Must(t1.New("foo").Parse(`bar`)) - var b bytes.Buffer + var b strings.Builder const wantError = `template: "foo" is an incomplete or empty template` if err := t1.Execute(&b, "javascript:alert(1)"); err == nil { t.Fatal("expected error executing t1") @@ -1981,7 +1981,7 @@ func TestAliasedParseTreeDoesNotOverescape(t *testing.T) { if _, err := tpl.AddParseTree("bar", tpl.Tree); err != nil { t.Fatalf("AddParseTree error: %v", err) } - var b1, b2 bytes.Buffer + var b1, b2 strings.Builder if err := tpl.ExecuteTemplate(&b1, "foo", data); err != nil { t.Fatalf(`ExecuteTemplate failed for "foo": %v`, err) } diff --git a/tpl/internal/go_templates/htmltemplate/exec_test.go b/tpl/internal/go_templates/htmltemplate/exec_test.go index 08195af0e..0f29cb060 100644 --- a/tpl/internal/go_templates/htmltemplate/exec_test.go +++ b/tpl/internal/go_templates/htmltemplate/exec_test.go @@ -769,7 +769,7 @@ func mapOfThree() any { } func testExecute(execTests []execTest, template *Template, t *testing.T) { - b := new(bytes.Buffer) + b := new(strings.Builder) funcs := FuncMap{ "add": add, "count": count, @@ -861,7 +861,7 @@ func TestDelims(t *testing.T) { if err != nil { t.Fatalf("delim %q text %q parse err %s", left, text, err) } - var b = new(bytes.Buffer) + var b = new(strings.Builder) err = tmpl.Execute(b, value) if err != nil { t.Fatalf("delim %q exec err %s", left, err) @@ -1002,7 +1002,7 @@ func TestTree(t *testing.T) { if err != nil { t.Fatal("parse error:", err) } - var b bytes.Buffer + var b strings.Builder const expect = "[1[2[3[4]][5[6]]][7[8[9]][10[11]]]]" // First by looking up the template. err = tmpl.Lookup("tree").Execute(&b, tree) @@ -1196,33 +1196,39 @@ var cmpTests = []cmpTest{ {"eq .Iface1 .Iface1", "true", true}, {"eq .Iface1 .Iface2", "false", true}, {"eq .Iface2 .Iface2", "true", true}, + {"eq .Map .Map", "true", true}, // Uncomparable types but nil is OK. + {"eq .Map nil", "true", true}, // Uncomparable types but nil is OK. + {"eq nil .Map", "true", true}, // Uncomparable types but nil is OK. + {"eq .Map .NonNilMap", "false", true}, // Uncomparable types but nil is OK. // Errors - {"eq `xy` 1", "", false}, // Different types. - {"eq 2 2.0", "", false}, // Different types. - {"lt true true", "", false}, // Unordered types. - {"lt 1+0i 1+0i", "", false}, // Unordered types. - {"eq .Ptr 1", "", false}, // Incompatible types. - {"eq .Ptr .NegOne", "", false}, // Incompatible types. - {"eq .Map .Map", "", false}, // Uncomparable types. - {"eq .Map .V1", "", false}, // Uncomparable types. + {"eq `xy` 1", "", false}, // Different types. + {"eq 2 2.0", "", false}, // Different types. + {"lt true true", "", false}, // Unordered types. + {"lt 1+0i 1+0i", "", false}, // Unordered types. + {"eq .Ptr 1", "", false}, // Incompatible types. + {"eq .Ptr .NegOne", "", false}, // Incompatible types. + {"eq .Map .V1", "", false}, // Uncomparable types. + {"eq .NonNilMap .NonNilMap", "", false}, // Uncomparable types. } func TestComparison(t *testing.T) { - b := new(bytes.Buffer) + b := new(strings.Builder) var cmpStruct = struct { Uthree, Ufour uint NegOne, Three int Ptr, NilPtr *int + NonNilMap map[int]int Map map[int]int V1, V2 V Iface1, Iface2 fmt.Stringer }{ - Uthree: 3, - Ufour: 4, - NegOne: -1, - Three: 3, - Ptr: new(int), - Iface1: b, + Uthree: 3, + Ufour: 4, + NegOne: -1, + Three: 3, + Ptr: new(int), + NonNilMap: make(map[int]int), + Iface1: b, } for _, test := range cmpTests { text := fmt.Sprintf("{{if %s}}true{{else}}false{{end}}", test.expr) @@ -1254,7 +1260,7 @@ func TestMissingMapKey(t *testing.T) { if err != nil { t.Fatal(err) } - var b bytes.Buffer + var b strings.Builder // By default, just get "<no value>" // NOTE: not in html/template, get empty string err = tmpl.Execute(&b, data) if err != nil { @@ -1423,7 +1429,7 @@ func TestBlock(t *testing.T) { t.Fatal(err) } - var buf bytes.Buffer + var buf strings.Builder if err := tmpl.Execute(&buf, "hello"); err != nil { t.Fatal(err) } @@ -1529,7 +1535,7 @@ func TestAddrOfIndex(t *testing.T) { } for _, text := range texts { tmpl := Must(New("tmpl").Parse(text)) - var buf bytes.Buffer + var buf strings.Builder err := tmpl.Execute(&buf, reflect.ValueOf([]V{{1}})) if err != nil { t.Fatalf("%s: Execute: %v", text, err) @@ -1585,7 +1591,7 @@ func TestInterfaceValues(t *testing.T) { for _, tt := range tests { tmpl := Must(New("tmpl").Parse(tt.text)) - var buf bytes.Buffer + var buf strings.Builder err := tmpl.Execute(&buf, map[string]any{ "PlusOne": func(n int) int { return n + 1 @@ -1680,7 +1686,7 @@ func TestIssue31810(t *testing.T) { t.Skip("broken in html/template") // A simple value with no arguments is fine. - var b bytes.Buffer + var b strings.Builder const text = "{{ (.) }}" tmpl, err := New("").Parse(text) if err != nil { diff --git a/tpl/internal/go_templates/htmltemplate/html.go b/tpl/internal/go_templates/htmltemplate/html.go index 19bd0ccb2..bcca0b51a 100644 --- a/tpl/internal/go_templates/htmltemplate/html.go +++ b/tpl/internal/go_templates/htmltemplate/html.go @@ -84,10 +84,12 @@ var htmlNormReplacementTable = []string{ // <script>(function () { // var a = [], d = document.getElementById("d"), i, c, s; // for (i = 0; i < 0x10000; ++i) { -// c = String.fromCharCode(i); -// d.innerHTML = "<span title=" + c + "lt" + c + "></span>" -// s = d.getElementsByTagName("SPAN")[0]; -// if (!s || s.title !== c + "lt" + c) { a.push(i.toString(16)); } +// +// c = String.fromCharCode(i); +// d.innerHTML = "<span title=" + c + "lt" + c + "></span>" +// s = d.getElementsByTagName("SPAN")[0]; +// if (!s || s.title !== c + "lt" + c) { a.push(i.toString(16)); } +// // } // document.write(a.join(", ")); // })()</script> @@ -174,7 +176,7 @@ func htmlReplacer(s string, replacementTable []string, badRunes bool) string { // stripTags takes a snippet of HTML and returns only the text content. // For example, `<b>¡Hi!</b> <script>...</script>` -> `¡Hi! `. func stripTags(html string) string { - var b bytes.Buffer + var b strings.Builder s, c, i, allText := []byte(html), context{}, 0, true // Using the transition funcs helps us avoid mangling // `<div title="1>2">` or `I <3 Ponies!`. diff --git a/tpl/internal/go_templates/htmltemplate/js_test.go b/tpl/internal/go_templates/htmltemplate/js_test.go index 92073b37a..483a3694f 100644 --- a/tpl/internal/go_templates/htmltemplate/js_test.go +++ b/tpl/internal/go_templates/htmltemplate/js_test.go @@ -8,7 +8,6 @@ package template import ( - "bytes" "math" "strings" "testing" @@ -324,7 +323,7 @@ func TestEscapersOnLower7AndSelectHighCodepoints(t *testing.T) { // Escape it rune by rune to make sure that any // fast-path checking does not break escaping. - var buf bytes.Buffer + var buf strings.Builder for _, c := range input { buf.WriteString(test.escaper(string(c))) } diff --git a/tpl/internal/go_templates/htmltemplate/multi_test.go b/tpl/internal/go_templates/htmltemplate/multi_test.go index 14cd7c766..f3e629b88 100644 --- a/tpl/internal/go_templates/htmltemplate/multi_test.go +++ b/tpl/internal/go_templates/htmltemplate/multi_test.go @@ -11,8 +11,8 @@ package template import ( "archive/zip" - "bytes" "os" + "strings" "testing" "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse" @@ -249,7 +249,7 @@ func TestEmptyTemplate(t *testing.T) { t.Fatal(err) } } - buf := &bytes.Buffer{} + buf := &strings.Builder{} if err := m.Execute(buf, c.in); err != nil { t.Error(i, err) continue @@ -284,7 +284,7 @@ func TestIssue19294(t *testing.T) { t.Fatal(err) } } - var buf bytes.Buffer + var buf strings.Builder res.Execute(&buf, 0) if buf.String() != "stylesheet" { t.Fatalf("iteration %d: got %q; expected %q", i, buf.String(), "stylesheet") diff --git a/tpl/internal/go_templates/htmltemplate/template.go b/tpl/internal/go_templates/htmltemplate/template.go index b4ccaa648..d7454f101 100644 --- a/tpl/internal/go_templates/htmltemplate/template.go +++ b/tpl/internal/go_templates/htmltemplate/template.go @@ -65,6 +65,7 @@ func (t *Template) Templates() []*Template { // // missingkey: Control the behavior during execution if a map is // indexed with a key that is not present in the map. +// // "missingkey=default" or "missingkey=invalid" // The default behavior: Do nothing and continue execution. // If printed, the result of the index operation is the string @@ -73,7 +74,6 @@ func (t *Template) Templates() []*Template { // The operation returns the zero value for the map type's element. // "missingkey=error" // Execution stops immediately with an error. -// func (t *Template) Option(opt ...string) *Template { t.text.Option(opt...) return t @@ -329,14 +329,7 @@ func (t *Template) Name() string { return t.text.Name() } -// FuncMap is the type of the map defining the mapping from names to -// functions. Each function must have either a single return value, or two -// return values of which the second has type error. In that case, if the -// second (error) argument evaluates to non-nil during execution, execution -// terminates and Execute returns that error. FuncMap has the same base type -// as FuncMap in "text/template", copied here so clients need not import -// "text/template". -type FuncMap map[string]any +type FuncMap = template.FuncMap // Funcs adds the elements of the argument map to the template's function map. // It must be called before the template is parsed. @@ -369,6 +362,7 @@ func (t *Template) Lookup(name string) *Template { // Must is a helper that wraps a call to a function returning (*Template, error) // and panics if the error is non-nil. It is intended for use in variable initializations // such as +// // var t = template.Must(template.New("name").Parse("html")) func Must(t *Template, err error) *Template { if err != nil { diff --git a/tpl/internal/go_templates/htmltemplate/template_test.go b/tpl/internal/go_templates/htmltemplate/template_test.go index 8a8f2f38c..0838308a1 100644 --- a/tpl/internal/go_templates/htmltemplate/template_test.go +++ b/tpl/internal/go_templates/htmltemplate/template_test.go @@ -30,7 +30,7 @@ func TestTemplateClone(t *testing.T) { const want = "stuff" parsed := Must(clone.Parse(want)) - var buf bytes.Buffer + var buf strings.Builder err = parsed.Execute(&buf, nil) if err != nil { t.Fatal(err) @@ -211,7 +211,7 @@ func (c *testCase) mustNotParse(t *Template, text string) { } func (c *testCase) mustExecute(t *Template, val any, want string) { - var buf bytes.Buffer + var buf strings.Builder err := t.Execute(&buf, val) if err != nil { c.t.Fatalf("execute: %v", err) diff --git a/tpl/internal/go_templates/htmltemplate/url.go b/tpl/internal/go_templates/htmltemplate/url.go index 93905586a..7820561dc 100644 --- a/tpl/internal/go_templates/htmltemplate/url.go +++ b/tpl/internal/go_templates/htmltemplate/url.go @@ -5,7 +5,6 @@ package template import ( - "bytes" "fmt" "strings" ) @@ -19,15 +18,15 @@ import ( // // This filter conservatively assumes that all schemes other than the following // are unsafe: -// * http: Navigates to a new website, and may open a new window or tab. -// These side effects can be reversed by navigating back to the -// previous website, or closing the window or tab. No irreversible -// changes will take place without further user interaction with -// the new website. -// * https: Same as http. -// * mailto: Opens an email program and starts a new draft. This side effect -// is not irreversible until the user explicitly clicks send; it -// can be undone by closing the email program. +// - http: Navigates to a new website, and may open a new window or tab. +// These side effects can be reversed by navigating back to the +// previous website, or closing the window or tab. No irreversible +// changes will take place without further user interaction with +// the new website. +// - https: Same as http. +// - mailto: Opens an email program and starts a new draft. This side effect +// is not irreversible until the user explicitly clicks send; it +// can be undone by closing the email program. // // To allow URLs containing other schemes to bypass this filter, developers must // explicitly indicate that such a URL is expected and safe by encapsulating it @@ -76,7 +75,7 @@ func urlProcessor(norm bool, args ...any) string { if t == contentTypeURL { norm = true } - var b bytes.Buffer + var b strings.Builder if processURLOnto(s, norm, &b) { return b.String() } @@ -85,7 +84,7 @@ func urlProcessor(norm bool, args ...any) string { // processURLOnto appends a normalized URL corresponding to its input to b // and reports whether the appended content differs from s. -func processURLOnto(s string, norm bool, b *bytes.Buffer) bool { +func processURLOnto(s string, norm bool, b *strings.Builder) bool { b.Grow(len(s) + 16) written := 0 // The byte loop below assumes that all URLs use UTF-8 as the @@ -149,7 +148,7 @@ func srcsetFilterAndEscaper(args ...any) string { case contentTypeURL: // Normalizing gets rid of all HTML whitespace // which separate the image URL from its metadata. - var b bytes.Buffer + var b strings.Builder if processURLOnto(s, true, &b) { s = b.String() } @@ -157,7 +156,7 @@ func srcsetFilterAndEscaper(args ...any) string { return strings.ReplaceAll(s, ",", "%2c") } - var b bytes.Buffer + var b strings.Builder written := 0 for i := 0; i < len(s); i++ { if s[i] == ',' { @@ -183,7 +182,7 @@ func isHTMLSpaceOrASCIIAlnum(c byte) bool { return (c < 0x80) && 0 != (htmlSpaceAndASCIIAlnumBytes[c>>3]&(1<<uint(c&0x7))) } -func filterSrcsetElement(s string, left int, right int, b *bytes.Buffer) { +func filterSrcsetElement(s string, left int, right int, b *strings.Builder) { start := left for start < right && isHTMLSpace(s[start]) { start++ diff --git a/tpl/internal/go_templates/testenv/exec.go b/tpl/internal/go_templates/testenv/exec.go new file mode 100644 index 000000000..cd06c4f7e --- /dev/null +++ b/tpl/internal/go_templates/testenv/exec.go @@ -0,0 +1,73 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testenv + +import ( + "os" + "os/exec" + "runtime" + "strings" + "sync" + "testing" +) + +// HasExec reports whether the current system can start new processes +// using os.StartProcess or (more commonly) exec.Command. +func HasExec() bool { + switch runtime.GOOS { + case "js", "ios": + return false + } + return true +} + +// MustHaveExec checks that the current system can start new processes +// using os.StartProcess or (more commonly) exec.Command. +// If not, MustHaveExec calls t.Skip with an explanation. +func MustHaveExec(t testing.TB) { + if !HasExec() { + t.Skipf("skipping test: cannot exec subprocess on %s/%s", runtime.GOOS, runtime.GOARCH) + } +} + +var execPaths sync.Map // path -> error + +// MustHaveExecPath checks that the current system can start the named executable +// using os.StartProcess or (more commonly) exec.Command. +// If not, MustHaveExecPath calls t.Skip with an explanation. +func MustHaveExecPath(t testing.TB, path string) { + MustHaveExec(t) + + err, found := execPaths.Load(path) + if !found { + _, err = exec.LookPath(path) + err, _ = execPaths.LoadOrStore(path, err) + } + if err != nil { + t.Skipf("skipping test: %s: %s", path, err) + } +} + +// CleanCmdEnv will fill cmd.Env with the environment, excluding certain +// variables that could modify the behavior of the Go tools such as +// GODEBUG and GOTRACEBACK. +func CleanCmdEnv(cmd *exec.Cmd) *exec.Cmd { + if cmd.Env != nil { + panic("environment already set") + } + for _, env := range os.Environ() { + // Exclude GODEBUG from the environment to prevent its output + // from breaking tests that are trying to parse other command output. + if strings.HasPrefix(env, "GODEBUG=") { + continue + } + // Exclude GOTRACEBACK for the same reason. + if strings.HasPrefix(env, "GOTRACEBACK=") { + continue + } + cmd.Env = append(cmd.Env, env) + } + return cmd +} diff --git a/tpl/internal/go_templates/testenv/noopt.go b/tpl/internal/go_templates/testenv/noopt.go new file mode 100644 index 000000000..ae2a3d011 --- /dev/null +++ b/tpl/internal/go_templates/testenv/noopt.go @@ -0,0 +1,12 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build noopt + +package testenv + +// OptimizationOff reports whether optimization is disabled. +func OptimizationOff() bool { + return true +} diff --git a/tpl/internal/go_templates/testenv/opt.go b/tpl/internal/go_templates/testenv/opt.go new file mode 100644 index 000000000..1bb96f73a --- /dev/null +++ b/tpl/internal/go_templates/testenv/opt.go @@ -0,0 +1,12 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !noopt + +package testenv + +// OptimizationOff reports whether optimization is disabled. +func OptimizationOff() bool { + return false +} diff --git a/tpl/internal/go_templates/testenv/testenv.go b/tpl/internal/go_templates/testenv/testenv.go index 510b5406e..4e38f5f04 100644 --- a/tpl/internal/go_templates/testenv/testenv.go +++ b/tpl/internal/go_templates/testenv/testenv.go @@ -11,10 +11,12 @@ package testenv import ( - "bytes" "errors" "flag" + "fmt" + "github.com/gohugoio/hugo/tpl/internal/go_templates/cfg" + "os" "os/exec" "path/filepath" @@ -23,7 +25,6 @@ import ( "strings" "sync" "testing" - "time" ) // Builder reports the name of the builder running this test @@ -34,7 +35,7 @@ func Builder() string { return os.Getenv("GO_BUILDER_NAME") } -// HasGoBuild reports whether the current system can build programs with ``go build'' +// HasGoBuild reports whether the current system can build programs with “go build” // and then run them with os.StartProcess or exec.Command. func HasGoBuild() bool { if os.Getenv("GO_GCFLAGS") != "" { @@ -51,7 +52,7 @@ func HasGoBuild() bool { return true } -// MustHaveGoBuild checks that the current system can build programs with ``go build'' +// MustHaveGoBuild checks that the current system can build programs with “go build” // and then run them with os.StartProcess or exec.Command. // If not, MustHaveGoBuild calls t.Skip with an explanation. func MustHaveGoBuild(t testing.TB) { @@ -63,13 +64,13 @@ func MustHaveGoBuild(t testing.TB) { } } -// HasGoRun reports whether the current system can run programs with ``go run.'' +// HasGoRun reports whether the current system can run programs with “go run.” func HasGoRun() bool { // For now, having go run and having go build are the same. return HasGoBuild() } -// MustHaveGoRun checks that the current system can run programs with ``go run.'' +// MustHaveGoRun checks that the current system can run programs with “go run.” // If not, MustHaveGoRun calls t.Skip with an explanation. func MustHaveGoRun(t testing.TB) { if !HasGoRun() { @@ -96,6 +97,100 @@ func GoToolPath(t testing.TB) string { return path } +var ( + gorootOnce sync.Once + gorootPath string + gorootErr error +) + +func findGOROOT() (string, error) { + gorootOnce.Do(func() { + gorootPath = runtime.GOROOT() + if gorootPath != "" { + // If runtime.GOROOT() is non-empty, assume that it is valid. + // + // (It might not be: for example, the user may have explicitly set GOROOT + // to the wrong directory, or explicitly set GOROOT_FINAL but not GOROOT + // and hasn't moved the tree to GOROOT_FINAL yet. But those cases are + // rare, and if that happens the user can fix what they broke.) + return + } + + // runtime.GOROOT doesn't know where GOROOT is (perhaps because the test + // binary was built with -trimpath, or perhaps because GOROOT_FINAL was set + // without GOROOT and the tree hasn't been moved there yet). + // + // Since this is internal/testenv, we can cheat and assume that the caller + // is a test of some package in a subdirectory of GOROOT/src. ('go test' + // runs the test in the directory containing the packaged under test.) That + // means that if we start walking up the tree, we should eventually find + // GOROOT/src/go.mod, and we can report the parent directory of that. + + cwd, err := os.Getwd() + if err != nil { + gorootErr = fmt.Errorf("finding GOROOT: %w", err) + return + } + + dir := cwd + for { + parent := filepath.Dir(dir) + if parent == dir { + // dir is either "." or only a volume name. + gorootErr = fmt.Errorf("failed to locate GOROOT/src in any parent directory") + return + } + + if base := filepath.Base(dir); base != "src" { + dir = parent + continue // dir cannot be GOROOT/src if it doesn't end in "src". + } + + b, err := os.ReadFile(filepath.Join(dir, "go.mod")) + if err != nil { + if os.IsNotExist(err) { + dir = parent + continue + } + gorootErr = fmt.Errorf("finding GOROOT: %w", err) + return + } + goMod := string(b) + + for goMod != "" { + var line string + line, goMod, _ = strings.Cut(goMod, "\n") + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "module" && fields[1] == "std" { + // Found "module std", which is the module declaration in GOROOT/src! + gorootPath = parent + return + } + } + } + }) + + return gorootPath, gorootErr +} + +// GOROOT reports the path to the directory containing the root of the Go +// project source tree. This is normally equivalent to runtime.GOROOT, but +// works even if the test binary was built with -trimpath. +// +// If GOROOT cannot be found, GOROOT skips t if t is non-nil, +// or panics otherwise. +func GOROOT(t testing.TB) string { + path, err := findGOROOT() + if err != nil { + if t == nil { + panic(err) + } + t.Helper() + t.Skip(err) + } + return path +} + // GoTool reports the path to the Go tool. func GoTool() (string, error) { if !HasGoBuild() { @@ -105,7 +200,11 @@ func GoTool() (string, error) { if runtime.GOOS == "windows" { exeSuffix = ".exe" } - path := filepath.Join(runtime.GOROOT(), "bin", "go"+exeSuffix) + goroot, err := findGOROOT() + if err != nil { + return "", fmt.Errorf("cannot find go tool: %w", err) + } + path := filepath.Join(goroot, "bin", "go"+exeSuffix) if _, err := os.Stat(path); err == nil { return path, nil } @@ -116,16 +215,6 @@ func GoTool() (string, error) { return goBin, nil } -// HasExec reports whether the current system can start new processes -// using os.StartProcess or (more commonly) exec.Command. -func HasExec() bool { - switch runtime.GOOS { - case "js", "ios": - return false - } - return true -} - // HasSrc reports whether the entire source tree is available under GOROOT. func HasSrc() bool { switch runtime.GOOS { @@ -135,33 +224,6 @@ func HasSrc() bool { return true } -// MustHaveExec checks that the current system can start new processes -// using os.StartProcess or (more commonly) exec.Command. -// If not, MustHaveExec calls t.Skip with an explanation. -func MustHaveExec(t testing.TB) { - if !HasExec() { - t.Skipf("skipping test: cannot exec subprocess on %s/%s", runtime.GOOS, runtime.GOARCH) - } -} - -var execPaths sync.Map // path -> error - -// MustHaveExecPath checks that the current system can start the named executable -// using os.StartProcess or (more commonly) exec.Command. -// If not, MustHaveExecPath calls t.Skip with an explanation. -func MustHaveExecPath(t testing.TB, path string) { - MustHaveExec(t) - - err, found := execPaths.Load(path) - if !found { - _, err = exec.LookPath(path) - err, _ = execPaths.LoadOrStore(path, err) - } - if err != nil { - t.Skipf("skipping test: %s: %s", path, err) - } -} - // HasExternalNetwork reports whether the current system can use // external (non-localhost) networks. func HasExternalNetwork() bool { @@ -194,32 +256,6 @@ func MustHaveCGO(t testing.TB) { } } -// CanInternalLink reports whether the current system can link programs with -// internal linking. -// (This is the opposite of cmd/internal/sys.MustLinkExternal. Keep them in sync.) -func CanInternalLink() bool { - switch runtime.GOOS { - case "android": - if runtime.GOARCH != "arm64" { - return false - } - case "ios": - if runtime.GOARCH == "arm64" { - return false - } - } - return true -} - -// MustInternalLink checks that the current system can link programs with internal -// linking. -// If not, MustInternalLink calls t.Skip with an explanation. -func MustInternalLink(t testing.TB) { - if !CanInternalLink() { - t.Skipf("skipping test: internal linking on %s/%s is not supported", runtime.GOOS, runtime.GOARCH) - } -} - // HasSymlink reports whether the current system can use os.Symlink. func HasSymlink() bool { ok, _ := hasSymlink() @@ -267,28 +303,6 @@ func SkipFlakyNet(t testing.TB) { } } -// CleanCmdEnv will fill cmd.Env with the environment, excluding certain -// variables that could modify the behavior of the Go tools such as -// GODEBUG and GOTRACEBACK. -func CleanCmdEnv(cmd *exec.Cmd) *exec.Cmd { - if cmd.Env != nil { - panic("environment already set") - } - for _, env := range os.Environ() { - // Exclude GODEBUG from the environment to prevent its output - // from breaking tests that are trying to parse other command output. - if strings.HasPrefix(env, "GODEBUG=") { - continue - } - // Exclude GOTRACEBACK for the same reason. - if strings.HasPrefix(env, "GOTRACEBACK=") { - continue - } - cmd.Env = append(cmd.Env, env) - } - return cmd -} - // CPUIsSlow reports whether the CPU running the test is suspected to be slow. func CPUIsSlow() bool { switch runtime.GOARCH { @@ -309,58 +323,10 @@ func SkipIfShortAndSlow(t testing.TB) { } } -// RunWithTimeout runs cmd and returns its combined output. If the -// subprocess exits with a non-zero status, it will log that status -// and return a non-nil error, but this is not considered fatal. -func RunWithTimeout(t testing.TB, cmd *exec.Cmd) ([]byte, error) { - args := cmd.Args - if args == nil { - args = []string{cmd.Path} - } - - var b bytes.Buffer - cmd.Stdout = &b - cmd.Stderr = &b - if err := cmd.Start(); err != nil { - t.Fatalf("starting %s: %v", args, err) - } - - // If the process doesn't complete within 1 minute, - // assume it is hanging and kill it to get a stack trace. - p := cmd.Process - done := make(chan bool) - go func() { - scale := 1 - // This GOARCH/GOOS test is copied from cmd/dist/test.go. - // TODO(iant): Have cmd/dist update the environment variable. - if runtime.GOARCH == "arm" || runtime.GOOS == "windows" { - scale = 2 - } - if s := os.Getenv("GO_TEST_TIMEOUT_SCALE"); s != "" { - if sc, err := strconv.Atoi(s); err == nil { - scale = sc - } - } - - select { - case <-done: - case <-time.After(time.Duration(scale) * time.Minute): - p.Signal(Sigquit) - // If SIGQUIT doesn't do it after a little - // while, kill the process. - select { - case <-done: - case <-time.After(time.Duration(scale) * 30 * time.Second): - p.Signal(os.Kill) - } - } - }() - - err := cmd.Wait() - if err != nil { - t.Logf("%s exit status: %v", args, err) +// SkipIfOptimizationOff skips t if optimization is disabled. +func SkipIfOptimizationOff(t testing.TB) { + if OptimizationOff() { + t.Helper() + t.Skip("skipping test with optimization disabled") } - close(done) - - return b.Bytes(), err } diff --git a/tpl/internal/go_templates/testenv/testenv_test.go b/tpl/internal/go_templates/testenv/testenv_test.go new file mode 100644 index 000000000..97c92a6e9 --- /dev/null +++ b/tpl/internal/go_templates/testenv/testenv_test.go @@ -0,0 +1,54 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testenv_test + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/gohugoio/hugo/tpl/internal/go_templates/testenv" +) + +func _TestGoToolLocation(t *testing.T) { + testenv.MustHaveGoBuild(t) + + var exeSuffix string + if runtime.GOOS == "windows" { + exeSuffix = ".exe" + } + + // Tests are defined to run within their package source directory, + // and this package's source directory is $GOROOT/src/internal/testenv. + // The 'go' command is installed at $GOROOT/bin/go, so if the environment + // is correct then testenv.GoTool() should be identical to ../../../bin/go. + + relWant := "../../../bin/go" + exeSuffix + absWant, err := filepath.Abs(relWant) + if err != nil { + t.Fatal(err) + } + + wantInfo, err := os.Stat(absWant) + if err != nil { + t.Fatal(err) + } + t.Logf("found go tool at %q (%q)", relWant, absWant) + + goTool, err := testenv.GoTool() + if err != nil { + t.Fatalf("testenv.GoTool(): %v", err) + } + t.Logf("testenv.GoTool() = %q", goTool) + + gotInfo, err := os.Stat(goTool) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(wantInfo, gotInfo) { + t.Fatalf("%q is not the same file as %q", absWant, goTool) + } +} diff --git a/tpl/internal/go_templates/testenv/testenv_unix.go b/tpl/internal/go_templates/testenv/testenv_unix.go index 3dc5daf45..a97e88da2 100644 --- a/tpl/internal/go_templates/testenv/testenv_unix.go +++ b/tpl/internal/go_templates/testenv/testenv_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris +//go:build unix package testenv diff --git a/tpl/internal/go_templates/texttemplate/doc.go b/tpl/internal/go_templates/texttemplate/doc.go index 10093881f..7817a17b9 100644 --- a/tpl/internal/go_templates/texttemplate/doc.go +++ b/tpl/internal/go_templates/texttemplate/doc.go @@ -18,7 +18,6 @@ structure as execution proceeds. The input text for a template is UTF-8-encoded text in any format. "Actions"--data evaluations or control structures--are delimited by "{{" and "}}"; all text outside actions is copied to the output unchanged. -Except for raw strings, actions may not span newlines, although comments can. Once parsed, a template may be executed safely in parallel, although if parallel executions share a Writer the output may be interleaved. @@ -425,10 +424,10 @@ The syntax of such definitions is to surround each template declaration with a The define action names the template being created by providing a string constant. Here is a simple example: - `{{define "T1"}}ONE{{end}} + {{define "T1"}}ONE{{end}} {{define "T2"}}TWO{{end}} {{define "T3"}}{{template "T1"}} {{template "T2"}}{{end}} - {{template "T3"}}` + {{template "T3"}} This defines two templates, T1 and T2, and a third T3 that invokes the other two when it is executed. Finally it invokes T3. If executed this template will diff --git a/tpl/internal/go_templates/texttemplate/exec.go b/tpl/internal/go_templates/texttemplate/exec.go index 4460771cb..62b2e519e 100644 --- a/tpl/internal/go_templates/texttemplate/exec.go +++ b/tpl/internal/go_templates/texttemplate/exec.go @@ -94,6 +94,12 @@ type missingValType struct{} var missingVal = reflect.ValueOf(missingValType{}) +var missingValReflectType = reflect.TypeOf(missingValType{}) + +func isMissing(v reflect.Value) bool { + return v.IsValid() && v.Type() == missingValReflectType +} + // at marks the state to be on node n, for error reporting. func (s *state) at(node parse.Node) { s.node = node @@ -357,11 +363,19 @@ func (s *state) walkRange(dot reflect.Value, r *parse.RangeNode) { oneIteration := func(index, elem reflect.Value) { // Set top var (lexically the second if there are two) to the element. if len(r.Pipe.Decl) > 0 { - s.setTopVar(1, elem) + if r.Pipe.IsAssign { + s.setVar(r.Pipe.Decl[0].Ident[0], elem) + } else { + s.setTopVar(1, elem) + } } // Set next var (lexically the first if there are two) to the index. if len(r.Pipe.Decl) > 1 { - s.setTopVar(2, index) + if r.Pipe.IsAssign { + s.setVar(r.Pipe.Decl[1].Ident[0], index) + } else { + s.setTopVar(2, index) + } } defer s.pop(mark) defer func() { @@ -471,7 +485,7 @@ func (s *state) evalPipeline(dot reflect.Value, pipe *parse.PipeNode) (value ref } func (s *state) notAFunction(args []parse.Node, final reflect.Value) { - if len(args) > 1 || final != missingVal { + if len(args) > 1 || !isMissing(final) { s.errorf("can't give argument to non-function %s", args[0]) } } @@ -629,7 +643,7 @@ func (s *state) evalFieldOld(dot reflect.Value, fieldName string, node parse.Nod if method := ptr.MethodByName(fieldName); method.IsValid() { return s.evalCall(dot, method, false, node, fieldName, args, final) } - hasArgs := len(args) > 1 || final != missingVal + hasArgs := len(args) > 1 || !isMissing(final) // It's not a method; must be a field of a struct or an element of a map. switch receiver.Kind() { case reflect.Struct: @@ -700,7 +714,7 @@ func (s *state) evalCallOld(dot, fun reflect.Value, isBuiltin bool, node parse.N } typ := fun.Type() numIn := len(args) - if final != missingVal { + if !isMissing(final) { numIn++ } numFixed := len(args) @@ -763,7 +777,7 @@ func (s *state) evalCallOld(dot, fun reflect.Value, isBuiltin bool, node parse.N } } // Add final value if necessary. - if final != missingVal { + if !isMissing(final) { t := typ.In(typ.NumIn() - 1) if typ.IsVariadic() { if numIn-1 < numFixed { diff --git a/tpl/internal/go_templates/texttemplate/exec_test.go b/tpl/internal/go_templates/texttemplate/exec_test.go index 64cb87ec6..45edb9e9b 100644 --- a/tpl/internal/go_templates/texttemplate/exec_test.go +++ b/tpl/internal/go_templates/texttemplate/exec_test.go @@ -695,6 +695,8 @@ var execTests = []execTest{ {"bug18a", "{{eq . '.'}}", "true", '.', true}, {"bug18b", "{{eq . 'e'}}", "true", 'e', true}, {"bug18c", "{{eq . 'P'}}", "true", 'P', true}, + + {"issue56490", "{{$i := 0}}{{$x := 0}}{{range $i = .AI}}{{end}}{{$i}}", "5", tVal, true}, } func zeroArgs() string { @@ -775,7 +777,7 @@ func mapOfThree() any { } func testExecute(execTests []execTest, template *Template, t *testing.T) { - b := new(bytes.Buffer) + b := new(strings.Builder) funcs := FuncMap{ "add": add, "count": count, @@ -864,7 +866,7 @@ func TestDelims(t *testing.T) { if err != nil { t.Fatalf("delim %q text %q parse err %s", left, text, err) } - var b = new(bytes.Buffer) + var b = new(strings.Builder) err = tmpl.Execute(b, value) if err != nil { t.Fatalf("delim %q exec err %s", left, err) @@ -1027,7 +1029,7 @@ func TestTree(t *testing.T) { if err != nil { t.Fatal("parse error:", err) } - var b bytes.Buffer + var b strings.Builder const expect = "[1[2[3[4]][5[6]]][7[8[9]][10[11]]]]" // First by looking up the template. err = tmpl.Lookup("tree").Execute(&b, tree) @@ -1223,33 +1225,39 @@ var cmpTests = []cmpTest{ {"eq .NilIface .Iface1", "false", true}, {"eq .NilIface 0", "false", true}, {"eq 0 .NilIface", "false", true}, + {"eq .Map .Map", "true", true}, // Uncomparable types but nil is OK. + {"eq .Map nil", "true", true}, // Uncomparable types but nil is OK. + {"eq nil .Map", "true", true}, // Uncomparable types but nil is OK. + {"eq .Map .NonNilMap", "false", true}, // Uncomparable types but nil is OK. // Errors - {"eq `xy` 1", "", false}, // Different types. - {"eq 2 2.0", "", false}, // Different types. - {"lt true true", "", false}, // Unordered types. - {"lt 1+0i 1+0i", "", false}, // Unordered types. - {"eq .Ptr 1", "", false}, // Incompatible types. - {"eq .Ptr .NegOne", "", false}, // Incompatible types. - {"eq .Map .Map", "", false}, // Uncomparable types. - {"eq .Map .V1", "", false}, // Uncomparable types. + {"eq `xy` 1", "", false}, // Different types. + {"eq 2 2.0", "", false}, // Different types. + {"lt true true", "", false}, // Unordered types. + {"lt 1+0i 1+0i", "", false}, // Unordered types. + {"eq .Ptr 1", "", false}, // Incompatible types. + {"eq .Ptr .NegOne", "", false}, // Incompatible types. + {"eq .Map .V1", "", false}, // Uncomparable types. + {"eq .NonNilMap .NonNilMap", "", false}, // Uncomparable types. } func TestComparison(t *testing.T) { - b := new(bytes.Buffer) + b := new(strings.Builder) var cmpStruct = struct { Uthree, Ufour uint NegOne, Three int Ptr, NilPtr *int + NonNilMap map[int]int Map map[int]int V1, V2 V Iface1, NilIface fmt.Stringer }{ - Uthree: 3, - Ufour: 4, - NegOne: -1, - Three: 3, - Ptr: new(int), - Iface1: b, + Uthree: 3, + Ufour: 4, + NegOne: -1, + Three: 3, + Ptr: new(int), + NonNilMap: make(map[int]int), + Iface1: b, } for _, test := range cmpTests { text := fmt.Sprintf("{{if %s}}true{{else}}false{{end}}", test.expr) @@ -1281,7 +1289,7 @@ func TestMissingMapKey(t *testing.T) { if err != nil { t.Fatal(err) } - var b bytes.Buffer + var b strings.Builder // By default, just get "<no value>" err = tmpl.Execute(&b, data) if err != nil { @@ -1451,7 +1459,7 @@ func TestBlock(t *testing.T) { t.Fatal(err) } - var buf bytes.Buffer + var buf strings.Builder if err := tmpl.Execute(&buf, "hello"); err != nil { t.Fatal(err) } @@ -1557,7 +1565,7 @@ func TestAddrOfIndex(t *testing.T) { } for _, text := range texts { tmpl := Must(New("tmpl").Parse(text)) - var buf bytes.Buffer + var buf strings.Builder err := tmpl.Execute(&buf, reflect.ValueOf([]V{{1}})) if err != nil { t.Fatalf("%s: Execute: %v", text, err) @@ -1613,7 +1621,7 @@ func TestInterfaceValues(t *testing.T) { for _, tt := range tests { tmpl := Must(New("tmpl").Parse(tt.text)) - var buf bytes.Buffer + var buf strings.Builder err := tmpl.Execute(&buf, map[string]any{ "PlusOne": func(n int) int { return n + 1 @@ -1706,7 +1714,7 @@ func TestExecutePanicDuringCall(t *testing.T) { // Issue 31810. Check that a parenthesized first argument behaves properly. func TestIssue31810(t *testing.T) { // A simple value with no arguments is fine. - var b bytes.Buffer + var b strings.Builder const text = "{{ (.) }}" tmpl, err := New("").Parse(text) if err != nil { diff --git a/tpl/internal/go_templates/texttemplate/funcs.go b/tpl/internal/go_templates/texttemplate/funcs.go index dca5ed28d..dbea6e705 100644 --- a/tpl/internal/go_templates/texttemplate/funcs.go +++ b/tpl/internal/go_templates/texttemplate/funcs.go @@ -5,7 +5,6 @@ package template import ( - "bytes" "errors" "fmt" "io" @@ -436,14 +435,33 @@ func basicKind(v reflect.Value) (kind, error) { return invalidKind, errBadComparisonType } +// isNil returns true if v is the zero reflect.Value, or nil of its type. +func isNil(v reflect.Value) bool { + if !v.IsValid() { + return true + } + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + return v.IsNil() + } + return false +} + +// canCompare reports whether v1 and v2 are both the same kind, or one is nil. +// Called only when dealing with nillable types, or there's about to be an error. +func canCompare(v1, v2 reflect.Value) bool { + k1 := v1.Kind() + k2 := v2.Kind() + if k1 == k2 { + return true + } + // We know the type can be compared to nil. + return k1 == reflect.Invalid || k2 == reflect.Invalid +} + // eq evaluates the comparison a == b || a == c || ... func eq(arg1 reflect.Value, arg2 ...reflect.Value) (bool, error) { arg1 = indirectInterface(arg1) - if arg1 != zero { - if t1 := arg1.Type(); !t1.Comparable() { - return false, fmt.Errorf("uncomparable type %s: %v", t1, arg1) - } - } if len(arg2) == 0 { return false, errNoComparison } @@ -479,11 +497,14 @@ func eq(arg1 reflect.Value, arg2 ...reflect.Value) (bool, error) { case uintKind: truth = arg1.Uint() == arg.Uint() default: - if arg == zero || arg1 == zero { - truth = arg1 == arg + if !canCompare(arg1, arg) { + return false, fmt.Errorf("non-comparable types %s: %v, %s: %v", arg1, arg1.Type(), arg.Type(), arg) + } + if isNil(arg1) || isNil(arg) { + truth = isNil(arg) == isNil(arg1) } else { - if t2 := arg.Type(); !t2.Comparable() { - return false, fmt.Errorf("uncomparable type %s: %v", t2, arg) + if !arg.Type().Comparable() { + return false, fmt.Errorf("non-comparable type %s: %v", arg, arg.Type()) } truth = arg1.Interface() == arg.Interface() } @@ -620,7 +641,7 @@ func HTMLEscapeString(s string) string { if !strings.ContainsAny(s, "'\"&<>\000") { return s } - var b bytes.Buffer + var b strings.Builder HTMLEscape(&b, []byte(s)) return b.String() } @@ -703,7 +724,7 @@ func JSEscapeString(s string) string { if strings.IndexFunc(s, jsIsSpecial) < 0 { return s } - var b bytes.Buffer + var b strings.Builder JSEscape(&b, []byte(s)) return b.String() } @@ -729,7 +750,9 @@ func URLQueryEscaper(args ...any) string { } // evalArgs formats the list of arguments into a string. It is therefore equivalent to +// // fmt.Sprint(args...) +// // except that each argument is indirected (if a pointer), as required, // using the same rules as the default string evaluation during template // execution. diff --git a/tpl/internal/go_templates/texttemplate/helper.go b/tpl/internal/go_templates/texttemplate/helper.go index 57905e613..48af3928b 100644 --- a/tpl/internal/go_templates/texttemplate/helper.go +++ b/tpl/internal/go_templates/texttemplate/helper.go @@ -19,6 +19,7 @@ import ( // Must is a helper that wraps a call to a function returning (*Template, error) // and panics if the error is non-nil. It is intended for use in variable // initializations such as +// // var t = template.Must(template.New("name").Parse("text")) func Must(t *Template, err error) *Template { if err != nil { diff --git a/tpl/internal/go_templates/texttemplate/multi_test.go b/tpl/internal/go_templates/texttemplate/multi_test.go index e3c9ec3ae..6e383c131 100644 --- a/tpl/internal/go_templates/texttemplate/multi_test.go +++ b/tpl/internal/go_templates/texttemplate/multi_test.go @@ -10,10 +10,10 @@ package template // Tests for multiple-template parsing and execution. import ( - "bytes" "fmt" "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse" "os" + "strings" "testing" ) @@ -245,7 +245,7 @@ func TestClone(t *testing.T) { } } // Execute root. - var b bytes.Buffer + var b strings.Builder err = root.ExecuteTemplate(&b, "a", 0) if err != nil { t.Fatal(err) @@ -284,7 +284,7 @@ func TestAddParseTree(t *testing.T) { t.Fatal(err) } // Execute. - var b bytes.Buffer + var b strings.Builder err = added.ExecuteTemplate(&b, "a", 0) if err != nil { t.Fatal(err) @@ -413,7 +413,7 @@ func TestEmptyTemplate(t *testing.T) { t.Fatal(err) } } - buf := &bytes.Buffer{} + buf := &strings.Builder{} if err := m.Execute(buf, c.in); err != nil { t.Error(i, err) continue @@ -448,7 +448,7 @@ func TestIssue19294(t *testing.T) { t.Fatal(err) } } - var buf bytes.Buffer + var buf strings.Builder res.Execute(&buf, 0) if buf.String() != "stylesheet" { t.Fatalf("iteration %d: got %q; expected %q", i, buf.String(), "stylesheet") diff --git a/tpl/internal/go_templates/texttemplate/option.go b/tpl/internal/go_templates/texttemplate/option.go index 1035afad7..ea2fd80c0 100644 --- a/tpl/internal/go_templates/texttemplate/option.go +++ b/tpl/internal/go_templates/texttemplate/option.go @@ -30,6 +30,7 @@ type option struct { // // missingkey: Control the behavior during execution if a map is // indexed with a key that is not present in the map. +// // "missingkey=default" or "missingkey=invalid" // The default behavior: Do nothing and continue execution. // If printed, the result of the index operation is the string @@ -38,7 +39,6 @@ type option struct { // The operation returns the zero value for the map type's element. // "missingkey=error" // Execution stops immediately with an error. -// func (t *Template) Option(opt ...string) *Template { t.init() for _, s := range opt { diff --git a/tpl/internal/go_templates/texttemplate/parse/lex.go b/tpl/internal/go_templates/texttemplate/parse/lex.go index 40d041112..3e60a1ece 100644 --- a/tpl/internal/go_templates/texttemplate/parse/lex.go +++ b/tpl/internal/go_templates/texttemplate/parse/lex.go @@ -111,31 +111,36 @@ type stateFn func(*lexer) stateFn // lexer holds the state of the scanner. type lexer struct { - name string // the name of the input; used only for error reports - input string // the string being scanned - leftDelim string // start of action - rightDelim string // end of action - emitComment bool // emit itemComment tokens. - pos Pos // current position in the input - start Pos // start position of this item - width Pos // width of last rune read from input - items chan item // channel of scanned items - parenDepth int // nesting depth of ( ) exprs - line int // 1+number of newlines seen - startLine int // start line of this item - breakOK bool // break keyword allowed - continueOK bool // continue keyword allowed + name string // the name of the input; used only for error reports + input string // the string being scanned + leftDelim string // start of action marker + rightDelim string // end of action marker + pos Pos // current position in the input + start Pos // start position of this item + atEOF bool // we have hit the end of input and returned eof + parenDepth int // nesting depth of ( ) exprs + line int // 1+number of newlines seen + startLine int // start line of this item + item item // item to return to parser + insideAction bool // are we inside an action? + options lexOptions +} + +// lexOptions control behavior of the lexer. All default to false. +type lexOptions struct { + emitComment bool // emit itemComment tokens. + breakOK bool // break keyword allowed + continueOK bool // continue keyword allowed } // next returns the next rune in the input. func (l *lexer) next() rune { if int(l.pos) >= len(l.input) { - l.width = 0 + l.atEOF = true return eof } r, w := utf8.DecodeRuneInString(l.input[l.pos:]) - l.width = Pos(w) - l.pos += l.width + l.pos += Pos(w) if r == '\n' { l.line++ } @@ -149,23 +154,41 @@ func (l *lexer) peek() rune { return r } -// backup steps back one rune. Can only be called once per call of next. +// backup steps back one rune. func (l *lexer) backup() { - l.pos -= l.width - // Correct newline count. - if l.width == 1 && l.input[l.pos] == '\n' { - l.line-- + if !l.atEOF && l.pos > 0 { + r, w := utf8.DecodeLastRuneInString(l.input[:l.pos]) + l.pos -= Pos(w) + // Correct newline count. + if r == '\n' { + l.line-- + } } } -// emit passes an item back to the client. -func (l *lexer) emit(t itemType) { - l.items <- item{t, l.start, l.input[l.start:l.pos], l.startLine} +// thisItem returns the item at the current input point with the specified type +// and advances the input. +func (l *lexer) thisItem(t itemType) item { + i := item{t, l.start, l.input[l.start:l.pos], l.startLine} l.start = l.pos l.startLine = l.line + return i +} + +// emit passes the trailing text as an item back to the parser. +func (l *lexer) emit(t itemType) stateFn { + return l.emitItem(l.thisItem(t)) +} + +// emitItem passes the specified item to the parser. +func (l *lexer) emitItem(i item) stateFn { + l.item = i + return nil } // ignore skips over the pending input before this point. +// It tracks newlines in the ignored text, so use it only +// for text that is skipped without calling l.next. func (l *lexer) ignore() { l.line += strings.Count(l.input[l.start:l.pos], "\n") l.start = l.pos @@ -191,25 +214,31 @@ func (l *lexer) acceptRun(valid string) { // errorf returns an error token and terminates the scan by passing // back a nil pointer that will be the next state, terminating l.nextItem. func (l *lexer) errorf(format string, args ...any) stateFn { - l.items <- item{itemError, l.start, fmt.Sprintf(format, args...), l.startLine} + l.item = item{itemError, l.start, fmt.Sprintf(format, args...), l.startLine} + l.start = 0 + l.pos = 0 + l.input = l.input[:0] return nil } // nextItem returns the next item from the input. // Called by the parser, not in the lexing goroutine. func (l *lexer) nextItem() item { - return <-l.items -} - -// drain drains the output so the lexing goroutine will exit. -// Called by the parser, not in the lexing goroutine. -func (l *lexer) drain() { - for range l.items { + l.item = item{itemEOF, l.pos, "EOF", l.startLine} + state := lexText + if l.insideAction { + state = lexInsideAction + } + for { + state = state(l) + if state == nil { + return l.item + } } } // lex creates a new scanner for the input string. -func lex(name, input, left, right string, emitComment bool) *lexer { +func lex(name, input, left, right string) *lexer { if left == "" { left = leftDelim } @@ -217,25 +246,15 @@ func lex(name, input, left, right string, emitComment bool) *lexer { right = rightDelim } l := &lexer{ - name: name, - input: input, - leftDelim: left, - rightDelim: right, - emitComment: emitComment, - items: make(chan item), - line: 1, - startLine: 1, - } - go l.run() - return l -} - -// run runs the state machine for the lexer. -func (l *lexer) run() { - for state := lexText; state != nil; { - state = state(l) + name: name, + input: input, + leftDelim: left, + rightDelim: right, + line: 1, + startLine: 1, + insideAction: false, } - close(l.items) + return l } // state functions @@ -249,31 +268,33 @@ const ( // lexText scans until an opening action delimiter, "{{". func lexText(l *lexer) stateFn { - l.width = 0 if x := strings.Index(l.input[l.pos:], l.leftDelim); x >= 0 { - ldn := Pos(len(l.leftDelim)) - l.pos += Pos(x) - trimLength := Pos(0) - if hasLeftTrimMarker(l.input[l.pos+ldn:]) { - trimLength = rightTrimLength(l.input[l.start:l.pos]) - } - l.pos -= trimLength - if l.pos > l.start { + if x > 0 { + l.pos += Pos(x) + // Do we trim any trailing space? + trimLength := Pos(0) + delimEnd := l.pos + Pos(len(l.leftDelim)) + if hasLeftTrimMarker(l.input[delimEnd:]) { + trimLength = rightTrimLength(l.input[l.start:l.pos]) + } + l.pos -= trimLength l.line += strings.Count(l.input[l.start:l.pos], "\n") - l.emit(itemText) + i := l.thisItem(itemText) + l.pos += trimLength + l.ignore() + if len(i.val) > 0 { + return l.emitItem(i) + } } - l.pos += trimLength - l.ignore() return lexLeftDelim } l.pos = Pos(len(l.input)) // Correctly reached EOF. if l.pos > l.start { l.line += strings.Count(l.input[l.start:l.pos], "\n") - l.emit(itemText) + return l.emit(itemText) } - l.emit(itemEOF) - return nil + return l.emit(itemEOF) } // rightTrimLength returns the length of the spaces at the end of the string. @@ -298,6 +319,7 @@ func leftTrimLength(s string) Pos { } // lexLeftDelim scans the left delimiter, which is known to be present, possibly with a trim marker. +// (The text to be trimmed has already been emitted.) func lexLeftDelim(l *lexer) stateFn { l.pos += Pos(len(l.leftDelim)) trimSpace := hasLeftTrimMarker(l.input[l.pos:]) @@ -310,28 +332,27 @@ func lexLeftDelim(l *lexer) stateFn { l.ignore() return lexComment } - l.emit(itemLeftDelim) + i := l.thisItem(itemLeftDelim) + l.insideAction = true l.pos += afterMarker l.ignore() l.parenDepth = 0 - return lexInsideAction + return l.emitItem(i) } // lexComment scans a comment. The left comment marker is known to be present. func lexComment(l *lexer) stateFn { l.pos += Pos(len(leftComment)) - i := strings.Index(l.input[l.pos:], rightComment) - if i < 0 { + x := strings.Index(l.input[l.pos:], rightComment) + if x < 0 { return l.errorf("unclosed comment") } - l.pos += Pos(i + len(rightComment)) + l.pos += Pos(x + len(rightComment)) delim, trimSpace := l.atRightDelim() if !delim { return l.errorf("comment ends before closing delimiter") } - if l.emitComment { - l.emit(itemComment) - } + i := l.thisItem(itemComment) if trimSpace { l.pos += trimMarkerLen } @@ -340,23 +361,27 @@ func lexComment(l *lexer) stateFn { l.pos += leftTrimLength(l.input[l.pos:]) } l.ignore() + if l.options.emitComment { + return l.emitItem(i) + } return lexText } // lexRightDelim scans the right delimiter, which is known to be present, possibly with a trim marker. func lexRightDelim(l *lexer) stateFn { - trimSpace := hasRightTrimMarker(l.input[l.pos:]) + _, trimSpace := l.atRightDelim() if trimSpace { l.pos += trimMarkerLen l.ignore() } l.pos += Pos(len(l.rightDelim)) - l.emit(itemRightDelim) + i := l.thisItem(itemRightDelim) if trimSpace { l.pos += leftTrimLength(l.input[l.pos:]) l.ignore() } - return lexText + l.insideAction = false + return l.emitItem(i) } // lexInsideAction scans the elements inside action delimiters. @@ -378,14 +403,14 @@ func lexInsideAction(l *lexer) stateFn { l.backup() // Put space back in case we have " -}}". return lexSpace case r == '=': - l.emit(itemAssign) + return l.emit(itemAssign) case r == ':': if l.next() != '=' { return l.errorf("expected :=") } - l.emit(itemDeclare) + return l.emit(itemDeclare) case r == '|': - l.emit(itemPipe) + return l.emit(itemPipe) case r == '"': return lexQuote case r == '`': @@ -410,20 +435,19 @@ func lexInsideAction(l *lexer) stateFn { l.backup() return lexIdentifier case r == '(': - l.emit(itemLeftParen) l.parenDepth++ + return l.emit(itemLeftParen) case r == ')': - l.emit(itemRightParen) l.parenDepth-- if l.parenDepth < 0 { - return l.errorf("unexpected right paren %#U", r) + return l.errorf("unexpected right paren") } + return l.emit(itemRightParen) case r <= unicode.MaxASCII && unicode.IsPrint(r): - l.emit(itemChar) + return l.emit(itemChar) default: return l.errorf("unrecognized character in action: %#U", r) } - return lexInsideAction } // lexSpace scans a run of space characters. @@ -448,13 +472,11 @@ func lexSpace(l *lexer) stateFn { return lexRightDelim // On the delim, so go right to that. } } - l.emit(itemSpace) - return lexInsideAction + return l.emit(itemSpace) } // lexIdentifier scans an alphanumeric. func lexIdentifier(l *lexer) stateFn { -Loop: for { switch r := l.next(); { case isAlphaNumeric(r): @@ -468,22 +490,19 @@ Loop: switch { case key[word] > itemKeyword: item := key[word] - if item == itemBreak && !l.breakOK || item == itemContinue && !l.continueOK { - l.emit(itemIdentifier) - } else { - l.emit(item) + if item == itemBreak && !l.options.breakOK || item == itemContinue && !l.options.continueOK { + return l.emit(itemIdentifier) } + return l.emit(item) case word[0] == '.': - l.emit(itemField) + return l.emit(itemField) case word == "true", word == "false": - l.emit(itemBool) + return l.emit(itemBool) default: - l.emit(itemIdentifier) + return l.emit(itemIdentifier) } - break Loop } } - return lexInsideAction } // lexField scans a field: .Alphanumeric. @@ -496,8 +515,7 @@ func lexField(l *lexer) stateFn { // The $ has been scanned. func lexVariable(l *lexer) stateFn { if l.atTerminator() { // Nothing interesting follows -> "$". - l.emit(itemVariable) - return lexInsideAction + return l.emit(itemVariable) } return lexFieldOrVariable(l, itemVariable) } @@ -507,11 +525,9 @@ func lexVariable(l *lexer) stateFn { func lexFieldOrVariable(l *lexer, typ itemType) stateFn { if l.atTerminator() { // Nothing interesting follows -> "." or "$". if typ == itemVariable { - l.emit(itemVariable) - } else { - l.emit(itemDot) + return l.emit(itemVariable) } - return lexInsideAction + return l.emit(itemDot) } var r rune for { @@ -524,8 +540,7 @@ func lexFieldOrVariable(l *lexer, typ itemType) stateFn { if !l.atTerminator() { return l.errorf("bad character %#U", r) } - l.emit(typ) - return lexInsideAction + return l.emit(typ) } // atTerminator reports whether the input is at valid termination character to @@ -541,13 +556,7 @@ func (l *lexer) atTerminator() bool { case eof, '.', ',', '|', ':', ')', '(': return true } - // Does r start the delimiter? This can be ambiguous (with delim=="//", $x/2 will - // succeed but should fail) but only in extremely rare cases caused by willfully - // bad choice of delimiter. - if rd, _ := utf8.DecodeRuneInString(l.rightDelim); rd == r { - return true - } - return false + return strings.HasPrefix(l.input[l.pos:], l.rightDelim) } // lexChar scans a character constant. The initial quote is already @@ -567,8 +576,7 @@ Loop: break Loop } } - l.emit(itemCharConstant) - return lexInsideAction + return l.emit(itemCharConstant) } // lexNumber scans a number: decimal, octal, hex, float, or imaginary. This @@ -584,11 +592,9 @@ func lexNumber(l *lexer) stateFn { if !l.scanNumber() || l.input[l.pos-1] != 'i' { return l.errorf("bad number syntax: %q", l.input[l.start:l.pos]) } - l.emit(itemComplex) - } else { - l.emit(itemNumber) + return l.emit(itemComplex) } - return lexInsideAction + return l.emit(itemNumber) } func (l *lexer) scanNumber() bool { @@ -644,8 +650,7 @@ Loop: break Loop } } - l.emit(itemString) - return lexInsideAction + return l.emit(itemString) } // lexRawQuote scans a raw quoted string. @@ -659,8 +664,7 @@ Loop: break Loop } } - l.emit(itemRawString) - return lexInsideAction + return l.emit(itemRawString) } // isSpace reports whether r is a space character. diff --git a/tpl/internal/go_templates/texttemplate/parse/lex_test.go b/tpl/internal/go_templates/texttemplate/parse/lex_test.go index 9189035fe..11e88792e 100644 --- a/tpl/internal/go_templates/texttemplate/parse/lex_test.go +++ b/tpl/internal/go_templates/texttemplate/parse/lex_test.go @@ -362,8 +362,7 @@ var lexTests = []lexTest{ {"extra right paren", "{{3)}}", []item{ tLeft, mkItem(itemNumber, "3"), - tRpar, - mkItem(itemError, `unexpected right paren U+0029 ')'`), + mkItem(itemError, "unexpected right paren"), }}, // Fixed bugs @@ -397,7 +396,12 @@ var lexTests = []lexTest{ // collect gathers the emitted items into a slice. func collect(t *lexTest, left, right string) (items []item) { - l := lex(t.name, t.input, left, right, true) + l := lex(t.name, t.input, left, right) + l.options = lexOptions{ + emitComment: true, + breakOK: true, + continueOK: true, + } for { item := l.nextItem() items = append(items, item) @@ -434,7 +438,9 @@ func TestLex(t *testing.T) { items := collect(&test, "", "") if !equal(items, test.items, false) { t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items) + return // TODO } + t.Log(test.name, "OK") } } @@ -472,6 +478,39 @@ func TestDelims(t *testing.T) { } } +func TestDelimsAlphaNumeric(t *testing.T) { + test := lexTest{"right delimiter with alphanumeric start", "{{hub .host hub}}", []item{ + mkItem(itemLeftDelim, "{{hub"), + mkItem(itemSpace, " "), + mkItem(itemField, ".host"), + mkItem(itemSpace, " "), + mkItem(itemRightDelim, "hub}}"), + tEOF, + }} + items := collect(&test, "{{hub", "hub}}") + + if !equal(items, test.items, false) { + t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items) + } +} + +func TestDelimsAndMarkers(t *testing.T) { + test := lexTest{"delims that look like markers", "{{- .x -}} {{- - .x - -}}", []item{ + mkItem(itemLeftDelim, "{{- "), + mkItem(itemField, ".x"), + mkItem(itemRightDelim, " -}}"), + mkItem(itemLeftDelim, "{{- "), + mkItem(itemField, ".x"), + mkItem(itemRightDelim, " -}}"), + tEOF, + }} + items := collect(&test, "{{- ", " -}}") + + if !equal(items, test.items, false) { + t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items) + } +} + var lexPosTests = []lexTest{ {"empty", "", []item{{itemEOF, 0, "", 1}}}, {"punctuation", "{{,@%#}}", []item{ @@ -533,22 +572,6 @@ func TestPos(t *testing.T) { } } -// Test that an error shuts down the lexing goroutine. -func TestShutdown(t *testing.T) { - // We need to duplicate template.Parse here to hold on to the lexer. - const text = "erroneous{{define}}{{else}}1234" - lexer := lex("foo", text, "{{", "}}", false) - _, err := New("root").parseLexer(lexer) - if err == nil { - t.Fatalf("expected error") - } - // The error should have drained the input. Therefore, the lexer should be shut down. - token, ok := <-lexer.items - if ok { - t.Fatalf("input was not drained; got %v", token) - } -} - // parseLexer is a local version of parse that lets us pass in the lexer instead of building it. // We expect an error, so the tree set and funcs list are explicitly nil. func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) { diff --git a/tpl/internal/go_templates/texttemplate/parse/parse.go b/tpl/internal/go_templates/texttemplate/parse/parse.go index ce548b088..87b7618f7 100644 --- a/tpl/internal/go_templates/texttemplate/parse/parse.go +++ b/tpl/internal/go_templates/texttemplate/parse/parse.go @@ -210,7 +210,6 @@ func (t *Tree) recover(errp *error) { panic(e) } if t != nil { - t.lex.drain() t.stopParse() } *errp = e.(error) @@ -224,8 +223,6 @@ func (t *Tree) startParse(funcs []map[string]any, lex *lexer, treeSet map[string t.vars = []string{"$"} t.funcs = funcs t.treeSet = treeSet - lex.breakOK = !t.hasFunction("break") - lex.continueOK = !t.hasFunction("continue") } // stopParse terminates parsing. @@ -243,8 +240,13 @@ func (t *Tree) stopParse() { func (t *Tree) Parse(text, leftDelim, rightDelim string, treeSet map[string]*Tree, funcs ...map[string]any) (tree *Tree, err error) { defer t.recover(&err) t.ParseName = t.Name - emitComment := t.Mode&ParseComments != 0 - t.startParse(funcs, lex(t.Name, text, leftDelim, rightDelim, emitComment), treeSet) + lexer := lex(t.Name, text, leftDelim, rightDelim) + lexer.options = lexOptions{ + emitComment: t.Mode&ParseComments != 0, + breakOK: !t.hasFunction("break"), + continueOK: !t.hasFunction("continue"), + } + t.startParse(funcs, lexer, treeSet) t.text = text t.parse() t.add() @@ -341,7 +343,9 @@ func (t *Tree) parseDefinition() { } // itemList: +// // textOrAction* +// // Terminates at {{end}} or {{else}}, returned separately. func (t *Tree) itemList() (list *ListNode, next Node) { list = t.newList(t.peekNonSpace().pos) @@ -358,6 +362,7 @@ func (t *Tree) itemList() (list *ListNode, next Node) { } // textOrAction: +// // text | comment | action func (t *Tree) textOrAction() Node { switch token := t.nextNonSpace(); token.typ { @@ -380,8 +385,10 @@ func (t *Tree) clearActionLine() { } // Action: +// // control // command ("|" command)* +// // Left delim is past. Now get actions. // First word could be a keyword such as range. func (t *Tree) action() (n Node) { @@ -412,7 +419,9 @@ func (t *Tree) action() (n Node) { } // Break: +// // {{break}} +// // Break keyword is past. func (t *Tree) breakControl(pos Pos, line int) Node { if token := t.nextNonSpace(); token.typ != itemRightDelim { @@ -425,7 +434,9 @@ func (t *Tree) breakControl(pos Pos, line int) Node { } // Continue: +// // {{continue}} +// // Continue keyword is past. func (t *Tree) continueControl(pos Pos, line int) Node { if token := t.nextNonSpace(); token.typ != itemRightDelim { @@ -438,6 +449,7 @@ func (t *Tree) continueControl(pos Pos, line int) Node { } // Pipeline: +// // declarations? command ('|' command)* func (t *Tree) pipeline(context string, end itemType) (pipe *PipeNode) { token := t.peekNonSpace() @@ -549,16 +561,20 @@ func (t *Tree) parseControl(allowElseIf bool, context string) (pos Pos, line int } // If: +// // {{if pipeline}} itemList {{end}} // {{if pipeline}} itemList {{else}} itemList {{end}} +// // If keyword is past. func (t *Tree) ifControl() Node { return t.newIf(t.parseControl(true, "if")) } // Range: +// // {{range pipeline}} itemList {{end}} // {{range pipeline}} itemList {{else}} itemList {{end}} +// // Range keyword is past. func (t *Tree) rangeControl() Node { r := t.newRange(t.parseControl(false, "range")) @@ -566,22 +582,28 @@ func (t *Tree) rangeControl() Node { } // With: +// // {{with pipeline}} itemList {{end}} // {{with pipeline}} itemList {{else}} itemList {{end}} +// // If keyword is past. func (t *Tree) withControl() Node { return t.newWith(t.parseControl(false, "with")) } // End: +// // {{end}} +// // End keyword is past. func (t *Tree) endControl() Node { return t.newEnd(t.expect(itemRightDelim, "end").pos) } // Else: +// // {{else}} +// // Else keyword is past. func (t *Tree) elseControl() Node { // Special case for "else if". @@ -595,7 +617,9 @@ func (t *Tree) elseControl() Node { } // Block: +// // {{block stringValue pipeline}} +// // Block keyword is past. // The name must be something that can evaluate to a string. // The pipeline is mandatory. @@ -623,7 +647,9 @@ func (t *Tree) blockControl() Node { } // Template: +// // {{template stringValue pipeline}} +// // Template keyword is past. The name must be something that can evaluate // to a string. func (t *Tree) templateControl() Node { @@ -654,7 +680,9 @@ func (t *Tree) parseTemplateName(token item, context string) (name string) { } // command: +// // operand (space operand)* +// // space-separated arguments up to a pipeline character or right delimiter. // we consume the pipe character but leave the right delim to terminate the action. func (t *Tree) command() *CommandNode { @@ -684,7 +712,9 @@ func (t *Tree) command() *CommandNode { } // operand: +// // term .Field* +// // An operand is a space-separated component of a command, // a term possibly followed by field accesses. // A nil return means the next item is not an operand. @@ -718,12 +748,14 @@ func (t *Tree) operand() Node { } // term: +// // literal (number, string, nil, boolean) // function (identifier) // . // .Field // $ // '(' pipeline ')' +// // A term is a simple "expression". // A nil return means the next item is not a term. func (t *Tree) term() Node { diff --git a/tpl/internal/go_templates/texttemplate/parse/parse_test.go b/tpl/internal/go_templates/texttemplate/parse/parse_test.go index 52bd6aca2..6ab48b3f7 100644 --- a/tpl/internal/go_templates/texttemplate/parse/parse_test.go +++ b/tpl/internal/go_templates/texttemplate/parse/parse_test.go @@ -492,7 +492,7 @@ var errorTests = []parseTest{ hasError, `unclosed left paren`}, {"rparen", "{{.X 1 2 3 ) }}", - hasError, `unexpected ")" in command`}, + hasError, "unexpected right paren"}, {"rparen2", "{{(.X 1 2 3", hasError, `unclosed action`}, @@ -600,7 +600,8 @@ func TestBlock(t *testing.T) { } func TestLineNum(t *testing.T) { - const count = 100 + // const count = 100 + const count = 3 text := strings.Repeat("{{printf 1234}}\n", count) tree, err := New("bench").Parse(text, "", "", make(map[string]*Tree), builtins) if err != nil { @@ -614,11 +615,11 @@ func TestLineNum(t *testing.T) { // Action first. action := nodes[i].(*ActionNode) if action.Line != line { - t.Fatalf("line %d: action is line %d", line, action.Line) + t.Errorf("line %d: action is line %d", line, action.Line) } pipe := action.Pipe if pipe.Line != line { - t.Fatalf("line %d: pipe is line %d", line, pipe.Line) + t.Errorf("line %d: pipe is line %d", line, pipe.Line) } } } |