Add Page.RenderShortcodes

A layouts/shortcodes/include.html shortcode may look like this:

```html
{{ $p := site.GetPage (.Get 0) }}
{{ $p.RenderShortcodes }}
```

Fixes #7297

9 files changed, 339 insertions, 20 deletions

diff --git a/hugolib/page.go b/hugolib/page.go
index f8f966156..81ba68aa4 100644
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -114,6 +114,10 @@ func (pa pageSiteAdapter) GetPage(ref string) (page.Page, error) {
 }
 
 type pageState struct {
+	// Incremented for each new page created.
+	// Note that this will change between builds for a given Page.
+	id int
+
 	// This slice will be of same length as the number of global slice of output
 	// formats (for all sites).
 	pageOutputs []*pageOutput
@@ -772,7 +776,7 @@ Loop:
 			currShortcode.pos = it.Pos()
 			currShortcode.length = iter.Current().Pos() - it.Pos()
 			if currShortcode.placeholder == "" {
-				currShortcode.placeholder = createShortcodePlaceholder("s", currShortcode.ordinal)
+				currShortcode.placeholder = createShortcodePlaceholder("s", p.id, currShortcode.ordinal)
 			}
 
 			if currShortcode.name != "" {
@@ -784,7 +788,7 @@ Loop:
 				currShortcode.params = s
 			}
 
-			currShortcode.placeholder = createShortcodePlaceholder("s", ordinal)
+			currShortcode.placeholder = createShortcodePlaceholder("s", p.id, ordinal)
 			ordinal++
 			s.shortcodes = append(s.shortcodes, currShortcode)
 
diff --git a/hugolib/page__new.go b/hugolib/page__new.go
index 14db28c3d..108e5717f 100644
--- a/hugolib/page__new.go
+++ b/hugolib/page__new.go
@@ -31,14 +31,19 @@ import (
 	"github.com/gohugoio/hugo/resources/page"
 )
 
+var pageIdCounter atomic.Int64
+
 func newPageBase(metaProvider *pageMeta) (*pageState, error) {
 	if metaProvider.s == nil {
 		panic("must provide a Site")
 	}
 
+	id := int(pageIdCounter.Add(1))
+
 	s := metaProvider.s
 
 	ps := &pageState{
+		id:                                id,
 		pageOutput:                        nopPageOutput,
 		pageOutputTemplateVariationsState: atomic.NewUint32(0),
 		pageCommon: &pageCommon{
diff --git a/hugolib/page__output.go b/hugolib/page__output.go
index 25ce26b7a..21f58e795 100644
--- a/hugolib/page__output.go
+++ b/hugolib/page__output.go
@@ -86,6 +86,7 @@ type pageOutput struct {
 	page.ContentProvider
 	page.PageRenderProvider
 	page.TableOfContentsProvider
+	page.RenderShortcodesProvider
 
 	// May be nil.
 	cp *pageContentOutput
@@ -99,6 +100,7 @@ func (p *pageOutput) initContentProvider(cp *pageContentOutput) {
 	p.ContentProvider = cp
 	p.PageRenderProvider = cp
 	p.TableOfContentsProvider = cp
+	p.RenderShortcodesProvider = cp
 	p.cp = cp
 
 }
diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go
index 89dc5ac77..e806ca339 100644
--- a/hugolib/page__per_output.go
+++ b/hugolib/page__per_output.go
@@ -103,6 +103,30 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err
 			return err
 		}
 
+		ctxCallback := func(cp2 *pageContentOutput) {
+			cp.p.cmap.hasNonMarkdownShortcode = cp.p.cmap.hasNonMarkdownShortcode || cp2.p.cmap.hasNonMarkdownShortcode
+			// Merge content placeholders
+			for k, v := range cp2.contentPlaceholders {
+				cp.contentPlaceholders[k] = v
+			}
+
+			if p.s.watching() {
+				for _, s := range cp2.p.shortcodeState.shortcodes {
+					for _, templ := range s.templs {
+						dependencyTracker.Add(templ.(identity.Manager))
+					}
+				}
+			}
+
+			// Transfer shortcode names so HasShortcode works for shortcodes from included pages.
+			cp.p.shortcodeState.transferNames(cp2.p.shortcodeState)
+			if cp2.p.pageOutputTemplateVariationsState.Load() == 2 {
+				cp.p.pageOutputTemplateVariationsState.Store(2)
+			}
+		}
+
+		ctx = tpl.SetCallbackFunctionInContext(ctx, ctxCallback)
+
 		var hasVariants bool
 		cp.workContent, hasVariants, err = p.contentToRender(ctx, p.source.parsed, p.cmap, cp.contentPlaceholders)
 		if err != nil {
@@ -350,6 +374,63 @@ func (p *pageContentOutput) Fragments(ctx context.Context) *tableofcontents.Frag
 	return p.tableOfContents
 }
 
+func (p *pageContentOutput) RenderShortcodes(ctx context.Context) (template.HTML, error) {
+	p.p.s.initInit(ctx, p.initToC, p.p)
+	source := p.p.source.parsed.Input()
+	renderedShortcodes := p.contentPlaceholders
+	var insertPlaceholders bool
+	var hasVariants bool
+	var cb func(*pageContentOutput)
+	if v := tpl.GetCallbackFunctionFromContext(ctx); v != nil {
+		if fn, ok := v.(func(*pageContentOutput)); ok {
+			insertPlaceholders = true
+			cb = fn
+		}
+	}
+	c := make([]byte, 0, len(source)+(len(source)/10))
+	for _, it := range p.p.cmap.items {
+		switch v := it.(type) {
+		case pageparser.Item:
+			c = append(c, source[v.Pos():v.Pos()+len(v.Val(source))]...)
+		case pageContentReplacement:
+			// Ignore.
+		case *shortcode:
+			if !insertPlaceholders || !v.insertPlaceholder() {
+				// Insert the rendered shortcode.
+				renderedShortcode, found := renderedShortcodes[v.placeholder]
+				if !found {
+					// This should never happen.
+					panic(fmt.Sprintf("rendered shortcode %q not found", v.placeholder))
+				}
+
+				b, more, err := renderedShortcode.renderShortcode(ctx)
+				if err != nil {
+					return "", fmt.Errorf("failed to render shortcode: %w", err)
+				}
+				hasVariants = hasVariants || more
+				c = append(c, []byte(b)...)
+
+			} else {
+				// Insert the placeholder so we can insert the content after
+				// markdown processing.
+				c = append(c, []byte(v.placeholder)...)
+			}
+		default:
+			panic(fmt.Sprintf("unknown item type %T", it))
+		}
+	}
+
+	if hasVariants {
+		p.p.pageOutputTemplateVariationsState.Store(2)
+	}
+
+	if cb != nil {
+		cb(p)
+	}
+
+	return helpers.BytesToHTML(c), nil
+}
+
 func (p *pageContentOutput) TableOfContents(ctx context.Context) template.HTML {
 	p.p.s.initInit(ctx, p.initToC, p.p)
 	return p.tableOfContentsHTML
diff --git a/hugolib/page_test.go b/hugolib/page_test.go
index 5237b6340..fd115385d 100644
--- a/hugolib/page_test.go
+++ b/hugolib/page_test.go
@@ -1998,7 +1998,6 @@ func TestRenderWithoutArgument(t *testing.T) {
 		IntegrationTestConfig{
 			T:           t,
 			TxtarString: files,
-			Running:     true,
 		},
 	).BuildE()
 
diff --git a/hugolib/rendershortcodes_test.go b/hugolib/rendershortcodes_test.go
new file mode 100644
index 000000000..c6fa711cc
--- /dev/null
+++ b/hugolib/rendershortcodes_test.go
@@ -0,0 +1,232 @@
+// Copyright 2023 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestRenderShortcodesBasic(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableKinds = ["home", "taxonomy", "term"]
+-- content/p1.md --
+---
+title: "p1"
+---
+## p1-h1
+{{% include "p2" %}}
+-- content/p2.md --
+---
+title: "p2"
+---
+### p2-h1
+{{< withhtml >}}
+### p2-h2
+{{% withmarkdown %}}
+### p2-h3
+{{% include "p3" %}}
+-- content/p3.md --
+---
+title: "p3"
+---
+### p3-h1
+{{< withhtml >}}
+### p3-h2
+{{% withmarkdown %}}
+{{< level3 >}}
+-- layouts/shortcodes/include.html --
+{{ $p := site.GetPage (.Get 0) }}
+{{ $p.RenderShortcodes }}
+-- layouts/shortcodes/withhtml.html --
+<div>{{ .Page.Title }} withhtml</div>
+-- layouts/shortcodes/withmarkdown.html --
+#### {{ .Page.Title }} withmarkdown
+-- layouts/shortcodes/level3.html --
+Level 3: {{ .Page.Title }}
+-- layouts/_default/single.html --
+Fragments: {{ .Fragments.Identifiers }}|
+HasShortcode Level 1: {{ .HasShortcode "include" }}|
+HasShortcode Level 2: {{ .HasShortcode "withmarkdown" }}|
+HasShortcode Level 3: {{ .HasShortcode "level3" }}|
+HasSHortcode not found: {{ .HasShortcode "notfound" }}|
+Content: {{ .Content }}|
+`
+
+	b := NewIntegrationTestBuilder(
+		IntegrationTestConfig{
+			T:           t,
+			TxtarString: files,
+		},
+	).Build()
+
+	b.AssertFileContent("public/p1/index.html",
+		"Fragments: [p1-h1 p2-h1 p2-h2 p2-h3 p2-withmarkdown p3-h1 p3-h2 p3-withmarkdown]|",
+		"HasShortcode Level 1: true|",
+		"HasShortcode Level 2: true|",
+		"HasShortcode Level 3: true|",
+		"HasSHortcode not found: false|",
+	)
+
+	// TODO1 more assertions.
+
+}
+
+func TestRenderShortcodesNestedMultipleOutputFormatTemplates(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableKinds = ["home", "taxonomy", "term", "section", "rss", "sitemap", "robotsTXT", "404"]
+[outputs]
+page = ["html", "json"]
+-- content/p1.md --
+---
+title: "p1"
+---
+## p1-h1
+{{% include "p2" %}}
+-- content/p2.md --
+---
+title: "p2"
+---
+### p2-h1
+{{% myshort %}}
+-- layouts/shortcodes/include.html --
+{{ $p := site.GetPage (.Get 0) }}
+{{ $p.RenderShortcodes }}
+-- layouts/shortcodes/myshort.html --
+Myshort HTML.
+-- layouts/shortcodes/myshort.json --
+Myshort JSON.
+-- layouts/_default/single.html --
+HTML: {{ .Content }}
+-- layouts/_default/single.json --
+JSON: {{ .Content }}
+
+
+`
+
+	b := NewIntegrationTestBuilder(
+		IntegrationTestConfig{
+			T:           t,
+			TxtarString: files,
+		},
+	).Build()
+
+	b.AssertFileContent("public/p1/index.html", "Myshort HTML")
+	b.AssertFileContent("public/p1/index.json", "Myshort JSON")
+
+}
+
+func TestRenderShortcodesEditNested(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableLiveReload = true
+disableKinds = ["home", "taxonomy", "term", "section", "rss", "sitemap", "robotsTXT", "404"]
+-- content/p1.md --
+---
+title: "p1"
+---
+## p1-h1
+{{% include "p2" %}}
+-- content/p2.md --
+---
+title: "p2"
+---
+### p2-h1
+{{% myshort %}}
+-- layouts/shortcodes/include.html --
+{{ $p := site.GetPage (.Get 0) }}
+{{ $p.RenderShortcodes }}
+-- layouts/shortcodes/myshort.html --
+Myshort Original.
+-- layouts/_default/single.html --
+ {{ .Content }}
+
+
+
+`
+
+	b := NewIntegrationTestBuilder(
+		IntegrationTestConfig{
+			T:           t,
+			TxtarString: files,
+			Running:     true,
+		},
+	).Build()
+
+	b.AssertFileContent("public/p1/index.html", "Myshort Original.")
+
+	b.EditFileReplace("layouts/shortcodes/myshort.html", func(s string) string {
+		return "Myshort Edited."
+	})
+	b.Build()
+	b.AssertFileContent("public/p1/index.html", "Myshort Edited.")
+
+}
+
+func TestRenderShortcodesEditIncludedPage(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableLiveReload = true
+disableKinds = ["home", "taxonomy", "term", "section", "rss", "sitemap", "robotsTXT", "404"]
+-- content/p1.md --
+---
+title: "p1"
+---
+## p1-h1
+{{% include "p2" %}}
+-- content/p2.md --
+---
+title: "p2"
+---
+### Original
+{{% myshort %}}
+-- layouts/shortcodes/include.html --
+{{ $p := site.GetPage (.Get 0) }}
+{{ $p.RenderShortcodes }}
+-- layouts/shortcodes/myshort.html --
+Myshort Original.
+-- layouts/_default/single.html --
+ {{ .Content }}
+
+
+
+`
+
+	b := NewIntegrationTestBuilder(
+		IntegrationTestConfig{
+			T:           t,
+			TxtarString: files,
+			Running:     true,
+		},
+	).Build()
+
+	b.AssertFileContent("public/p1/index.html", "Original")
+
+	b.EditFileReplace("content/p2.md", func(s string) string {
+		return strings.Replace(s, "Original", "Edited", 1)
+	})
+	b.Build()
+	b.AssertFileContent("public/p1/index.html", "Edited")
+
+}
diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go
index c12fb888b..e201d4d6b 100644
--- a/hugolib/shortcode.go
+++ b/hugolib/shortcode.go
@@ -185,8 +185,8 @@ func (scp *ShortcodeWithPage) page() page.Page {
 // Note - this value must not contain any markup syntax
 const shortcodePlaceholderPrefix = "HAHAHUGOSHORTCODE"
 
-func createShortcodePlaceholder(id string, ordinal int) string {
-	return shortcodePlaceholderPrefix + id + strconv.Itoa(ordinal) + "HBHB"
+func createShortcodePlaceholder(sid string, id, ordinal int) string {
+	return shortcodePlaceholderPrefix + strconv.Itoa(id) + sid + strconv.Itoa(ordinal) + "HBHB"
 }
 
 type shortcode struct {
diff --git a/hugolib/shortcode_page.go b/hugolib/shortcode_page.go
index 20fa22d2f..f351daae0 100644
--- a/hugolib/shortcode_page.go
+++ b/hugolib/shortcode_page.go
@@ -21,7 +21,7 @@ import (
 )
 
 // A placeholder for the TableOfContents markup. This is what we pass to the Goldmark etc. renderers.
-var tocShortcodePlaceholder = createShortcodePlaceholder("TOC", 0)
+var tocShortcodePlaceholder = createShortcodePlaceholder("TOC", 0, 0)
 
 // shortcodeRenderer is typically used to delay rendering of inner shortcodes
 // marked with placeholders in the content.
diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go
index 774794c56..6ee68673f 100644
--- a/hugolib/shortcode_test.go
+++ b/hugolib/shortcode_test.go
@@ -948,7 +948,6 @@ title: "p1"
 		IntegrationTestConfig{
 			T:           t,
 			TxtarString: files,
-			Running:     true,
 		},
 	).Build()
 
@@ -991,7 +990,6 @@ title: "p1"
 		IntegrationTestConfig{
 			T:           t,
 			TxtarString: files,
-			Running:     true,
 		},
 	).Build()
 
@@ -1023,7 +1021,6 @@ echo "foo";
 		IntegrationTestConfig{
 			T:           t,
 			TxtarString: files,
-			Running:     true,
 		},
 	).Build()
 
@@ -1061,7 +1058,6 @@ title: "p1"
 		IntegrationTestConfig{
 			T:           t,
 			TxtarString: files,
-			Running:     true,
 		},
 	).Build()
 
@@ -1098,8 +1094,8 @@ Title: {{ .Get "title" | safeHTML }}
 		IntegrationTestConfig{
 			T:           t,
 			TxtarString: files,
-			Running:     true,
-			Verbose:     true,
+
+			Verbose: true,
 		},
 	).Build()
 
@@ -1191,8 +1187,8 @@ C'est un test
 		IntegrationTestConfig{
 			T:           t,
 			TxtarString: files,
-			Running:     true,
-			Verbose:     true,
+
+			Verbose: true,
 		},
 	).Build()
 
@@ -1229,8 +1225,8 @@ InnerDeindent: {{ .Get 0 }}: {{ len .InnerDeindent }}
 		IntegrationTestConfig{
 			T:           t,
 			TxtarString: files,
-			Running:     true,
-			Verbose:     true,
+
+			Verbose: true,
 		},
 	).Build()
 
@@ -1269,8 +1265,8 @@ Inner: {{ .Get 0 }}: {{ len .Inner }}
 		IntegrationTestConfig{
 			T:           t,
 			TxtarString: files,
-			Running:     true,
-			Verbose:     true,
+
+			Verbose: true,
 		},
 	).BuildE()
 
@@ -1306,8 +1302,8 @@ Hello.
 		IntegrationTestConfig{
 			T:           t,
 			TxtarString: files,
-			Running:     true,
-			Verbose:     true,
+
+			Verbose: true,
 		},
 	).Build()