Add Page.Contents with scope support

Note that this also adds a new `.ContentWithoutSummary` method, and to do that we had to unify the different summary types:

Both `auto` and `manual` now returns HTML. Before this commit, `auto` would return plain text. This could be considered to be a slightly breaking change, but for the better: Now you can treat the `.Summary` the same without thinking about where it comes from, and if you want plain text, pipe it into `{{ .Summary | plainify }}`.

Fixes #8680
Fixes #12761
Fixes #12778
Fixes #716

1 files changed, 70 insertions, 322 deletions

diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go
index 59cb574df..f074e8db7 100644
--- a/hugolib/page__per_output.go
+++ b/hugolib/page__per_output.go
@@ -21,18 +21,14 @@ import (
 	"html/template"
 	"strings"
 	"sync"
+	"sync/atomic"
 
+	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/common/text"
-	"github.com/gohugoio/hugo/common/types/hstring"
 	"github.com/gohugoio/hugo/identity"
-	"github.com/gohugoio/hugo/markup"
-	"github.com/gohugoio/hugo/media"
-	"github.com/gohugoio/hugo/parser/pageparser"
-	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/cast"
 
 	"github.com/gohugoio/hugo/markup/converter/hooks"
-	"github.com/gohugoio/hugo/markup/goldmark/hugocontext"
 	"github.com/gohugoio/hugo/markup/highlight/chromalexers"
 	"github.com/gohugoio/hugo/markup/tableofcontents"
 
@@ -41,7 +37,6 @@ import (
 	bp "github.com/gohugoio/hugo/bufferpool"
 	"github.com/gohugoio/hugo/tpl"
 
-	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/output"
 	"github.com/gohugoio/hugo/resources/page"
 	"github.com/gohugoio/hugo/resources/resource"
@@ -73,7 +68,7 @@ func newPageContentOutput(po *pageOutput) (*pageContentOutput, error) {
 	cp := &pageContentOutput{
 		po:           po,
 		renderHooks:  &renderHooks{},
-		otherOutputs: make(map[uint64]*pageContentOutput),
+		otherOutputs: maps.NewCache[uint64, *pageContentOutput](),
 	}
 	return cp, nil
 }
@@ -89,10 +84,10 @@ type pageContentOutput struct {
 
 	// Other pages involved in rendering of this page,
 	// typically included with .RenderShortcodes.
-	otherOutputs map[uint64]*pageContentOutput
+	otherOutputs *maps.Cache[uint64, *pageContentOutput]
 
-	contentRenderedVersion uint32 // Incremented on reset.
-	contentRendered        bool   // Set on content render.
+	contentRenderedVersion uint32      // Incremented on reset.
+	contentRendered        atomic.Bool // Set on content render.
 
 	// Renders Markdown hooks.
 	renderHooks *renderHooks
@@ -107,109 +102,84 @@ func (pco *pageContentOutput) Reset() {
 		return
 	}
 	pco.contentRenderedVersion++
-	pco.contentRendered = false
+	pco.contentRendered.Store(false)
 	pco.renderHooks = &renderHooks{}
 }
 
-func (pco *pageContentOutput) Fragments(ctx context.Context) *tableofcontents.Fragments {
-	return pco.po.p.m.content.mustContentToC(ctx, pco).tableOfContents
-}
-
-func (pco *pageContentOutput) RenderShortcodes(ctx context.Context) (template.HTML, error) {
-	content := pco.po.p.m.content
-	source, err := content.pi.contentSource(content)
-	if err != nil {
-		return "", err
+func (pco *pageContentOutput) Render(ctx context.Context, layout ...string) (template.HTML, error) {
+	if len(layout) == 0 {
+		return "", errors.New("no layout given")
 	}
-	ct, err := content.contentToC(ctx, pco)
+	templ, found, err := pco.po.p.resolveTemplate(layout...)
 	if err != nil {
-		return "", err
+		return "", pco.po.p.wrapError(err)
 	}
 
-	var insertPlaceholders bool
-	var hasVariants bool
-	cb := setGetContentCallbackInContext.Get(ctx)
-	if cb != nil {
-		insertPlaceholders = true
+	if !found {
+		return "", nil
 	}
-	c := make([]byte, 0, len(source)+(len(source)/10))
-	for _, it := range content.pi.itemsStep2 {
-		switch v := it.(type) {
-		case pageparser.Item:
-			c = append(c, source[v.Pos():v.Pos()+len(v.Val(source))]...)
-		case pageContentReplacement:
-			// Ignore.
-		case *shortcode:
-			if !insertPlaceholders || !v.insertPlaceholder() {
-				// Insert the rendered shortcode.
-				renderedShortcode, found := ct.contentPlaceholders[v.placeholder]
-				if !found {
-					// This should never happen.
-					panic(fmt.Sprintf("rendered shortcode %q not found", v.placeholder))
-				}
 
-				b, more, err := renderedShortcode.renderShortcode(ctx)
-				if err != nil {
-					return "", fmt.Errorf("failed to render shortcode: %w", err)
-				}
-				hasVariants = hasVariants || more
-				c = append(c, []byte(b)...)
-
-			} else {
-				// Insert the placeholder so we can insert the content after
-				// markdown processing.
-				c = append(c, []byte(v.placeholder)...)
-			}
-		default:
-			panic(fmt.Sprintf("unknown item type %T", it))
-		}
+	// Make sure to send the *pageState and not the *pageContentOutput to the template.
+	res, err := executeToString(ctx, pco.po.p.s.Tmpl(), templ, pco.po.p)
+	if err != nil {
+		return "", pco.po.p.wrapError(fmt.Errorf("failed to execute template %s: %w", templ.Name(), err))
 	}
+	return template.HTML(res), nil
+}
 
-	if hasVariants {
-		pco.po.p.pageOutputTemplateVariationsState.Add(1)
-	}
+func (pco *pageContentOutput) Fragments(ctx context.Context) *tableofcontents.Fragments {
+	return pco.c().Fragments(ctx)
+}
 
-	if cb != nil {
-		cb(pco, ct)
-	}
+func (pco *pageContentOutput) RenderShortcodes(ctx context.Context) (template.HTML, error) {
+	return pco.c().RenderShortcodes(ctx)
+}
 
-	if tpl.Context.IsInGoldmark.Get(ctx) {
-		// This content will be parsed and rendered by Goldmark.
-		// Wrap it in a special Hugo markup to assign the correct Page from
-		// the stack.
-		return template.HTML(hugocontext.Wrap(c, pco.po.p.pid)), nil
+func (pco *pageContentOutput) Markup(opts ...any) page.Markup {
+	if len(opts) > 1 {
+		panic("too many arguments, expected 0 or 1")
+	}
+	var scope string
+	if len(opts) == 1 {
+		scope = cast.ToString(opts[0])
 	}
+	return pco.po.p.m.content.getOrCreateScope(scope, pco)
+}
 
-	return helpers.BytesToHTML(c), nil
+func (pco *pageContentOutput) c() page.Markup {
+	return pco.po.p.m.content.getOrCreateScope("", pco)
 }
 
 func (pco *pageContentOutput) Content(ctx context.Context) (any, error) {
-	r, err := pco.po.p.m.content.contentRendered(ctx, pco)
-	return r.content, err
+	r, err := pco.c().Render(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return r.Content(ctx)
 }
 
-func (pco *pageContentOutput) TableOfContents(ctx context.Context) template.HTML {
-	return pco.po.p.m.content.mustContentToC(ctx, pco).tableOfContentsHTML
+func (pco *pageContentOutput) ContentWithoutSummary(ctx context.Context) (template.HTML, error) {
+	r, err := pco.c().Render(ctx)
+	if err != nil {
+		return "", err
+	}
+	return r.ContentWithoutSummary(ctx)
 }
 
-func (p *pageContentOutput) Len(ctx context.Context) int {
-	return len(p.mustContentRendered(ctx).content)
+func (pco *pageContentOutput) TableOfContents(ctx context.Context) template.HTML {
+	return pco.c().(*cachedContentScope).fragmentsHTML(ctx)
 }
 
-func (pco *pageContentOutput) mustContentRendered(ctx context.Context) contentSummary {
-	r, err := pco.po.p.m.content.contentRendered(ctx, pco)
-	if err != nil {
-		pco.fail(err)
-	}
-	return r
+func (pco *pageContentOutput) Len(ctx context.Context) int {
+	return pco.mustRender(ctx).Len(ctx)
 }
 
-func (pco *pageContentOutput) mustContentPlain(ctx context.Context) contentPlainPlainWords {
-	r, err := pco.po.p.m.content.contentPlain(ctx, pco)
+func (pco *pageContentOutput) mustRender(ctx context.Context) page.Content {
+	c, err := pco.c().Render(ctx)
 	if err != nil {
 		pco.fail(err)
 	}
-	return r
+	return c
 }
 
 func (pco *pageContentOutput) fail(err error) {
@@ -217,203 +187,43 @@ func (pco *pageContentOutput) fail(err error) {
 }
 
 func (pco *pageContentOutput) Plain(ctx context.Context) string {
-	return pco.mustContentPlain(ctx).plain
+	return pco.mustRender(ctx).Plain(ctx)
 }
 
 func (pco *pageContentOutput) PlainWords(ctx context.Context) []string {
-	return pco.mustContentPlain(ctx).plainWords
+	return pco.mustRender(ctx).PlainWords(ctx)
 }
 
 func (pco *pageContentOutput) ReadingTime(ctx context.Context) int {
-	return pco.mustContentPlain(ctx).readingTime
+	return pco.mustRender(ctx).ReadingTime(ctx)
 }
 
 func (pco *pageContentOutput) WordCount(ctx context.Context) int {
-	return pco.mustContentPlain(ctx).wordCount
+	return pco.mustRender(ctx).WordCount(ctx)
 }
 
 func (pco *pageContentOutput) FuzzyWordCount(ctx context.Context) int {
-	return pco.mustContentPlain(ctx).fuzzyWordCount
+	return pco.mustRender(ctx).FuzzyWordCount(ctx)
 }
 
 func (pco *pageContentOutput) Summary(ctx context.Context) template.HTML {
-	return pco.mustContentPlain(ctx).summary
-}
-
-func (pco *pageContentOutput) Truncated(ctx context.Context) bool {
-	return pco.mustContentPlain(ctx).summaryTruncated
-}
-
-func (pco *pageContentOutput) RenderString(ctx context.Context, args ...any) (template.HTML, error) {
-	if len(args) < 1 || len(args) > 2 {
-		return "", errors.New("want 1 or 2 arguments")
-	}
-
-	var contentToRender string
-	opts := defaultRenderStringOpts
-	sidx := 1
-
-	if len(args) == 1 {
-		sidx = 0
-	} else {
-		m, ok := args[0].(map[string]any)
-		if !ok {
-			return "", errors.New("first argument must be a map")
-		}
-
-		if err := mapstructure.WeakDecode(m, &opts); err != nil {
-			return "", fmt.Errorf("failed to decode options: %w", err)
-		}
-		if opts.Markup != "" {
-			opts.Markup = markup.ResolveMarkup(opts.Markup)
-		}
-	}
-
-	contentToRenderv := args[sidx]
-
-	if _, ok := contentToRenderv.(hstring.RenderedString); ok {
-		// This content is already rendered, this is potentially
-		// a infinite recursion.
-		return "", errors.New("text is already rendered, repeating it may cause infinite recursion")
-	}
-
-	var err error
-	contentToRender, err = cast.ToStringE(contentToRenderv)
+	summary, err := pco.mustRender(ctx).Summary(ctx)
 	if err != nil {
-		return "", err
-	}
-
-	if err = pco.initRenderHooks(); err != nil {
-		return "", err
-	}
-
-	conv := pco.po.p.getContentConverter()
-
-	if opts.Markup != "" && opts.Markup != pco.po.p.m.pageConfig.ContentMediaType.SubType {
-		var err error
-		conv, err = pco.po.p.m.newContentConverter(pco.po.p, opts.Markup)
-		if err != nil {
-			return "", pco.po.p.wrapError(err)
-		}
-	}
-
-	var rendered []byte
-
-	parseInfo := &contentParseInfo{
-		h:   pco.po.p.s.h,
-		pid: pco.po.p.pid,
-	}
-
-	if pageparser.HasShortcode(contentToRender) {
-		contentToRenderb := []byte(contentToRender)
-		// String contains a shortcode.
-		parseInfo.itemsStep1, err = pageparser.ParseBytes(contentToRenderb, pageparser.Config{
-			NoFrontMatter:    true,
-			NoSummaryDivider: true,
-		})
-		if err != nil {
-			return "", err
-		}
-
-		s := newShortcodeHandler(pco.po.p.pathOrTitle(), pco.po.p.s)
-		if err := parseInfo.mapItemsAfterFrontMatter(contentToRenderb, s); err != nil {
-			return "", err
-		}
-
-		placeholders, err := s.prepareShortcodesForPage(ctx, pco.po.p, pco.po.f, true)
-		if err != nil {
-			return "", err
-		}
-
-		contentToRender, hasVariants, err := parseInfo.contentToRender(ctx, contentToRenderb, placeholders)
-		if err != nil {
-			return "", err
-		}
-		if hasVariants {
-			pco.po.p.pageOutputTemplateVariationsState.Add(1)
-		}
-		b, err := pco.renderContentWithConverter(ctx, conv, contentToRender, false)
-		if err != nil {
-			return "", pco.po.p.wrapError(err)
-		}
-		rendered = b.Bytes()
-
-		if parseInfo.hasNonMarkdownShortcode {
-			var hasShortcodeVariants bool
-
-			tokenHandler := func(ctx context.Context, token string) ([]byte, error) {
-				if token == tocShortcodePlaceholder {
-					toc, err := pco.po.p.m.content.contentToC(ctx, pco)
-					if err != nil {
-						return nil, err
-					}
-					// The Page's TableOfContents was accessed in a shortcode.
-					return []byte(toc.tableOfContentsHTML), nil
-				}
-				renderer, found := placeholders[token]
-				if found {
-					repl, more, err := renderer.renderShortcode(ctx)
-					if err != nil {
-						return nil, err
-					}
-					hasShortcodeVariants = hasShortcodeVariants || more
-					return repl, nil
-				}
-				// This should not happen.
-				return nil, fmt.Errorf("unknown shortcode token %q", token)
-			}
-
-			rendered, err = expandShortcodeTokens(ctx, rendered, tokenHandler)
-			if err != nil {
-				return "", err
-			}
-			if hasShortcodeVariants {
-				pco.po.p.pageOutputTemplateVariationsState.Add(1)
-			}
-		}
-
-		// We need a consolidated view in $page.HasShortcode
-		pco.po.p.m.content.shortcodeState.transferNames(s)
-
-	} else {
-		c, err := pco.renderContentWithConverter(ctx, conv, []byte(contentToRender), false)
-		if err != nil {
-			return "", pco.po.p.wrapError(err)
-		}
-
-		rendered = c.Bytes()
-	}
-
-	if opts.Display == "inline" {
-		markup := pco.po.p.m.pageConfig.Content.Markup
-		if opts.Markup != "" {
-			markup = pco.po.p.s.ContentSpec.ResolveMarkup(opts.Markup)
-		}
-		rendered = pco.po.p.s.ContentSpec.TrimShortHTML(rendered, markup)
+		pco.fail(err)
 	}
-
-	return template.HTML(string(rendered)), nil
+	return summary.Text
 }
 
-func (pco *pageContentOutput) Render(ctx context.Context, layout ...string) (template.HTML, error) {
-	if len(layout) == 0 {
-		return "", errors.New("no layout given")
-	}
-	templ, found, err := pco.po.p.resolveTemplate(layout...)
+func (pco *pageContentOutput) Truncated(ctx context.Context) bool {
+	summary, err := pco.mustRender(ctx).Summary(ctx)
 	if err != nil {
-		return "", pco.po.p.wrapError(err)
-	}
-
-	if !found {
-		return "", nil
+		pco.fail(err)
 	}
+	return summary.Truncated
+}
 
-	// Make sure to send the *pageState and not the *pageContentOutput to the template.
-	res, err := executeToString(ctx, pco.po.p.s.Tmpl(), templ, pco.po.p)
-	if err != nil {
-		return "", pco.po.p.wrapError(fmt.Errorf("failed to execute template %s: %w", templ.Name(), err))
-	}
-	return template.HTML(res), nil
+func (pco *pageContentOutput) RenderString(ctx context.Context, args ...any) (template.HTML, error) {
+	return pco.c().RenderString(ctx, args...)
 }
 
 func (pco *pageContentOutput) initRenderHooks() error {
@@ -660,65 +470,3 @@ func executeToString(ctx context.Context, h tpl.TemplateHandler, templ tpl.Templ
 	}
 	return b.String(), nil
 }
-
-func splitUserDefinedSummaryAndContent(markup string, c []byte) (summary []byte, content []byte, err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			err = fmt.Errorf("summary split failed: %s", r)
-		}
-	}()
-
-	startDivider := bytes.Index(c, internalSummaryDividerBaseBytes)
-
-	if startDivider == -1 {
-		return
-	}
-
-	startTag := "p"
-	switch markup {
-	case media.DefaultContentTypes.AsciiDoc.SubType:
-		startTag = "div"
-	}
-
-	// Walk back and forward to the surrounding tags.
-	start := bytes.LastIndex(c[:startDivider], []byte("<"+startTag))
-	end := bytes.Index(c[startDivider:], []byte("</"+startTag))
-
-	if start == -1 {
-		start = startDivider
-	} else {
-		start = startDivider - (startDivider - start)
-	}
-
-	if end == -1 {
-		end = startDivider + len(internalSummaryDividerBase)
-	} else {
-		end = startDivider + end + len(startTag) + 3
-	}
-
-	var addDiv bool
-
-	switch markup {
-	case "rst":
-		addDiv = true
-	}
-
-	withoutDivider := append(c[:start], bytes.Trim(c[end:], "\n")...)
-
-	if len(withoutDivider) > 0 {
-		summary = bytes.TrimSpace(withoutDivider[:start])
-	}
-
-	if addDiv {
-		// For the rst
-		summary = append(append([]byte(nil), summary...), []byte("</div>")...)
-	}
-
-	if err != nil {
-		return
-	}
-
-	content = bytes.TrimSpace(withoutDivider)
-
-	return
-}