Make Page an interface

The main motivation of this commit is to add a `page.Page` interface to replace the very file-oriented `hugolib.Page` struct.
This is all a preparation step for issue  #5074, "pages from other data sources".

But this also fixes a set of annoying limitations, especially related to custom output formats, and shortcodes.

Most notable changes:

* The inner content of shortcodes using the `{{%` as the outer-most delimiter will now be sent to the content renderer, e.g. Blackfriday.
  This means that any markdown will partake in the global ToC and footnote context etc.
* The Custom Output formats are now "fully virtualized". This removes many of the current limitations.
* The taxonomy list type now has a reference to the `Page` object.
  This improves the taxonomy template `.Title` situation and make common template constructs much simpler.

See #5074
Fixes #5763
Fixes #5758
Fixes #5090
Fixes #5204
Fixes #4695
Fixes #5607
Fixes #5707
Fixes #5719
Fixes #3113
Fixes #5706
Fixes #5767
Fixes #5723
Fixes #5769
Fixes #5770
Fixes #5771
Fixes #5759
Fixes #5776
Fixes #5777
Fixes #5778

1 files changed, 323 insertions, 186 deletions

diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go
index 83b96b7f4..236fd11a6 100644
--- a/hugolib/hugo_sites_build_test.go
+++ b/hugolib/hugo_sites_build_test.go
@@ -1,16 +1,16 @@
 package hugolib
 
 import (
-	"bytes"
 	"fmt"
 	"strings"
 	"testing"
 
-	"html/template"
 	"os"
 	"path/filepath"
 	"time"
 
+	"github.com/gohugoio/hugo/resources/page"
+
 	"github.com/fortytw2/leaktest"
 	"github.com/fsnotify/fsnotify"
 	"github.com/gohugoio/hugo/helpers"
@@ -66,8 +66,8 @@ func doTestMultiSitesMainLangInRoot(t *testing.T, defaultInSubDir bool) {
 
 	assert.Equal("/blog/en/foo", enSite.PathSpec.RelURL("foo", true))
 
-	doc1en := enSite.RegularPages[0]
-	doc1fr := frSite.RegularPages[0]
+	doc1en := enSite.RegularPages()[0]
+	doc1fr := frSite.RegularPages()[0]
 
 	enPerm := doc1en.Permalink()
 	enRelPerm := doc1en.RelPermalink()
@@ -100,7 +100,7 @@ func doTestMultiSitesMainLangInRoot(t *testing.T, defaultInSubDir bool) {
 	// Check list pages
 	b.AssertFileContent(pathMod("public/fr/sect/index.html"), "List", "Bonjour")
 	b.AssertFileContent("public/en/sect/index.html", "List", "Hello")
-	b.AssertFileContent(pathMod("public/fr/plaques/frtag1/index.html"), "Taxonomy List", "Bonjour")
+	b.AssertFileContent(pathMod("public/fr/plaques/FRtag1/index.html"), "Taxonomy List", "Bonjour")
 	b.AssertFileContent("public/en/tags/tag1/index.html", "Taxonomy List", "Hello")
 
 	// Check sitemaps
@@ -126,8 +126,8 @@ func doTestMultiSitesMainLangInRoot(t *testing.T, defaultInSubDir bool) {
 		pathMod(`<atom:link href="http://example.com/blog/fr/sect/index.xml"`))
 	b.AssertFileContent("public/en/sect/index.xml", `<atom:link href="http://example.com/blog/en/sect/index.xml"`)
 	b.AssertFileContent(
-		pathMod("public/fr/plaques/frtag1/index.xml"),
-		pathMod(`<atom:link href="http://example.com/blog/fr/plaques/frtag1/index.xml"`))
+		pathMod("public/fr/plaques/FRtag1/index.xml"),
+		pathMod(`<atom:link href="http://example.com/blog/fr/plaques/FRtag1/index.xml"`))
 	b.AssertFileContent("public/en/tags/tag1/index.xml", `<atom:link href="http://example.com/blog/en/tags/tag1/index.xml"`)
 
 	// Check paginators
@@ -140,12 +140,12 @@ func doTestMultiSitesMainLangInRoot(t *testing.T, defaultInSubDir bool) {
 	b.AssertFileContent(pathMod("public/fr/sect/page/2/index.html"), "List Page 2", "Bonjour", pathMod("http://example.com/blog/fr/sect/"))
 	b.AssertFileContent("public/en/sect/page/2/index.html", "List Page 2", "Hello", "http://example.com/blog/en/sect/")
 	b.AssertFileContent(
-		pathMod("public/fr/plaques/frtag1/page/1/index.html"),
-		pathMod(`refresh" content="0; url=http://example.com/blog/fr/plaques/frtag1/"`))
+		pathMod("public/fr/plaques/FRtag1/page/1/index.html"),
+		pathMod(`refresh" content="0; url=http://example.com/blog/fr/plaques/FRtag1/"`))
 	b.AssertFileContent("public/en/tags/tag1/page/1/index.html", `refresh" content="0; url=http://example.com/blog/en/tags/tag1/"`)
 	b.AssertFileContent(
-		pathMod("public/fr/plaques/frtag1/page/2/index.html"), "List Page 2", "Bonjour",
-		pathMod("http://example.com/blog/fr/plaques/frtag1/"))
+		pathMod("public/fr/plaques/FRtag1/page/2/index.html"), "List Page 2", "Bonjour",
+		pathMod("http://example.com/blog/fr/plaques/FRtag1/"))
 	b.AssertFileContent("public/en/tags/tag1/page/2/index.html", "List Page 2", "Hello", "http://example.com/blog/en/tags/tag1/")
 	// nn (Nynorsk) and nb (Bokmål) have custom pagePath: side ("page" in Norwegian)
 	b.AssertFileContent("public/nn/side/1/index.html", `refresh" content="0; url=http://example.com/blog/nn/"`)
@@ -183,12 +183,12 @@ p1 = "p1en"
 	assert.Len(sites, 2)
 
 	nnSite := sites[0]
-	nnHome := nnSite.getPage(KindHome)
+	nnHome := nnSite.getPage(page.KindHome)
 	assert.Len(nnHome.AllTranslations(), 2)
 	assert.Len(nnHome.Translations(), 1)
 	assert.True(nnHome.IsTranslated())
 
-	enHome := sites[1].getPage(KindHome)
+	enHome := sites[1].getPage(page.KindHome)
 
 	p1, err := enHome.Param("p1")
 	assert.NoError(err)
@@ -199,9 +199,7 @@ p1 = "p1en"
 	assert.Equal("p1nn", p1)
 }
 
-//
 func TestMultiSitesBuild(t *testing.T) {
-	t.Parallel()
 
 	for _, config := range []struct {
 		content string
@@ -211,7 +209,11 @@ func TestMultiSitesBuild(t *testing.T) {
 		{multiSiteYAMLConfigTemplate, "yml"},
 		{multiSiteJSONConfigTemplate, "json"},
 	} {
-		doTestMultiSitesBuild(t, config.content, config.suffix)
+
+		t.Run(config.suffix, func(t *testing.T) {
+			t.Parallel()
+			doTestMultiSitesBuild(t, config.content, config.suffix)
+		})
 	}
 }
 
@@ -228,64 +230,51 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) {
 
 	// Check site config
 	for _, s := range sites {
-		require.True(t, s.Info.defaultContentLanguageInSubdir, s.Info.Title)
+		require.True(t, s.Info.defaultContentLanguageInSubdir, s.Info.title)
 		require.NotNil(t, s.disabledKinds)
 	}
 
 	gp1 := b.H.GetContentPage(filepath.FromSlash("content/sect/doc1.en.md"))
 	require.NotNil(t, gp1)
-	require.Equal(t, "doc1", gp1.title)
+	require.Equal(t, "doc1", gp1.Title())
 	gp2 := b.H.GetContentPage(filepath.FromSlash("content/dummysect/notfound.md"))
 	require.Nil(t, gp2)
 
 	enSite := sites[0]
-	enSiteHome := enSite.getPage(KindHome)
+	enSiteHome := enSite.getPage(page.KindHome)
 	require.True(t, enSiteHome.IsTranslated())
 
-	require.Equal(t, "en", enSite.Language.Lang)
-
-	assert.Equal(5, len(enSite.RegularPages))
-	assert.Equal(32, len(enSite.AllPages))
+	require.Equal(t, "en", enSite.language.Lang)
 
-	doc1en := enSite.RegularPages[0]
-	permalink := doc1en.Permalink()
-	require.Equal(t, "http://example.com/blog/en/sect/doc1-slug/", permalink, "invalid doc1.en permalink")
-	require.Len(t, doc1en.Translations(), 1, "doc1-en should have one translation, excluding itself")
+	assert.Equal(5, len(enSite.RegularPages()))
+	assert.Equal(32, len(enSite.AllPages()))
 
-	doc2 := enSite.RegularPages[1]
-	permalink = doc2.Permalink()
-	require.Equal(t, "http://example.com/blog/en/sect/doc2/", permalink, "invalid doc2 permalink")
+	// Check 404s
+	b.AssertFileContent("public/en/404.html", "404|en|404 Page not found")
+	b.AssertFileContent("public/fr/404.html", "404|fr|404 Page not found")
 
-	doc3 := enSite.RegularPages[2]
-	permalink = doc3.Permalink()
-	// Note that /superbob is a custom URL set in frontmatter.
-	// We respect that URL literally (it can be /search.json)
-	// and do no not do any language code prefixing.
-	require.Equal(t, "http://example.com/blog/superbob/", permalink, "invalid doc3 permalink")
+	// Check robots.txt
+	b.AssertFileContent("public/en/robots.txt", "robots|en|")
+	b.AssertFileContent("public/nn/robots.txt", "robots|nn|")
 
-	require.Equal(t, "/superbob", doc3.URL(), "invalid url, was specified on doc3")
-	b.AssertFileContent("public/superbob/index.html", "doc3|Hello|en")
-	require.Equal(t, doc2.PrevPage, doc3, "doc3 should follow doc2, in .PrevPage")
+	b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Permalink: http://example.com/blog/en/sect/doc1-slug/")
+	b.AssertFileContent("public/en/sect/doc2/index.html", "Permalink: http://example.com/blog/en/sect/doc2/")
+	b.AssertFileContent("public/superbob/index.html", "Permalink: http://example.com/blog/superbob/")
 
+	doc2 := enSite.RegularPages()[1]
+	doc3 := enSite.RegularPages()[2]
+	require.Equal(t, doc2.Prev(), doc3, "doc3 should follow doc2, in .PrevPage")
+	doc1en := enSite.RegularPages()[0]
 	doc1fr := doc1en.Translations()[0]
-	permalink = doc1fr.Permalink()
-	require.Equal(t, "http://example.com/blog/fr/sect/doc1/", permalink, "invalid doc1fr permalink")
+	b.AssertFileContent("public/fr/sect/doc1/index.html", "Permalink: http://example.com/blog/fr/sect/doc1/")
 
 	require.Equal(t, doc1en.Translations()[0], doc1fr, "doc1-en should have doc1-fr as translation")
 	require.Equal(t, doc1fr.Translations()[0], doc1en, "doc1-fr should have doc1-en as translation")
 	require.Equal(t, "fr", doc1fr.Language().Lang)
 
-	doc4 := enSite.AllPages[4]
-	permalink = doc4.Permalink()
-	require.Equal(t, "http://example.com/blog/fr/sect/doc4/", permalink, "invalid doc4 permalink")
-	require.Equal(t, "/blog/fr/sect/doc4/", doc4.URL())
-
+	doc4 := enSite.AllPages()[4]
 	require.Len(t, doc4.Translations(), 0, "found translations for doc4")
 
-	doc5 := enSite.AllPages[5]
-	permalink = doc5.Permalink()
-	require.Equal(t, "http://example.com/blog/fr/somewhere/else/doc5/", permalink, "invalid doc5 permalink")
-
 	// Taxonomies and their URLs
 	require.Len(t, enSite.Taxonomies, 1, "should have 1 taxonomy")
 	tags := enSite.Taxonomies["tags"]
@@ -294,12 +283,13 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) {
 
 	frSite := sites[1]
 
-	require.Equal(t, "fr", frSite.Language.Lang)
-	require.Len(t, frSite.RegularPages, 4, "should have 3 pages")
-	require.Len(t, frSite.AllPages, 32, "should have 32 total pages (including translations and nodes)")
+	require.Equal(t, "fr", frSite.language.Lang)
+	require.Len(t, frSite.RegularPages(), 4, "should have 3 pages")
+	require.Len(t, frSite.AllPages(), 32, "should have 32 total pages (including translations and nodes)")
 
-	for _, frenchPage := range frSite.RegularPages {
-		require.Equal(t, "fr", frenchPage.Lang())
+	for _, frenchPage := range frSite.RegularPages() {
+		p := frenchPage
+		require.Equal(t, "fr", p.Language().Lang)
 	}
 
 	// See https://github.com/gohugoio/hugo/issues/4285
@@ -307,10 +297,10 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) {
 	// isn't ideal in a multilingual setup. You want a way to get the current language version if available.
 	// Now you can do lookups with translation base name to get that behaviour.
 	// Let us test all the regular page variants:
-	getPageDoc1En := enSite.getPage(KindPage, filepath.ToSlash(doc1en.Path()))
-	getPageDoc1EnBase := enSite.getPage(KindPage, "sect/doc1")
-	getPageDoc1Fr := frSite.getPage(KindPage, filepath.ToSlash(doc1fr.Path()))
-	getPageDoc1FrBase := frSite.getPage(KindPage, "sect/doc1")
+	getPageDoc1En := enSite.getPage(page.KindPage, filepath.ToSlash(doc1en.File().Path()))
+	getPageDoc1EnBase := enSite.getPage(page.KindPage, "sect/doc1")
+	getPageDoc1Fr := frSite.getPage(page.KindPage, filepath.ToSlash(doc1fr.File().Path()))
+	getPageDoc1FrBase := frSite.getPage(page.KindPage, "sect/doc1")
 	require.Equal(t, doc1en, getPageDoc1En)
 	require.Equal(t, doc1fr, getPageDoc1Fr)
 	require.Equal(t, doc1en, getPageDoc1EnBase)
@@ -328,35 +318,36 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) {
 	b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Single", "Shortcode: Hello", "LingoDefault")
 
 	// Check node translations
-	homeEn := enSite.getPage(KindHome)
+	homeEn := enSite.getPage(page.KindHome)
 	require.NotNil(t, homeEn)
 	require.Len(t, homeEn.Translations(), 3)
-	require.Equal(t, "fr", homeEn.Translations()[0].Lang())
-	require.Equal(t, "nn", homeEn.Translations()[1].Lang())
-	require.Equal(t, "På nynorsk", homeEn.Translations()[1].title)
-	require.Equal(t, "nb", homeEn.Translations()[2].Lang())
-	require.Equal(t, "På bokmål", homeEn.Translations()[2].title, configSuffix)
+	require.Equal(t, "fr", homeEn.Translations()[0].Language().Lang)
+	require.Equal(t, "nn", homeEn.Translations()[1].Language().Lang)
+	require.Equal(t, "På nynorsk", homeEn.Translations()[1].Title())
+	require.Equal(t, "nb", homeEn.Translations()[2].Language().Lang)
+	require.Equal(t, "På bokmål", homeEn.Translations()[2].Title(), configSuffix)
 	require.Equal(t, "Bokmål", homeEn.Translations()[2].Language().LanguageName, configSuffix)
 
-	sectFr := frSite.getPage(KindSection, "sect")
+	sectFr := frSite.getPage(page.KindSection, "sect")
 	require.NotNil(t, sectFr)
 
-	require.Equal(t, "fr", sectFr.Lang())
+	require.Equal(t, "fr", sectFr.Language().Lang)
 	require.Len(t, sectFr.Translations(), 1)
-	require.Equal(t, "en", sectFr.Translations()[0].Lang())
-	require.Equal(t, "Sects", sectFr.Translations()[0].title)
+	require.Equal(t, "en", sectFr.Translations()[0].Language().Lang)
+	require.Equal(t, "Sects", sectFr.Translations()[0].Title())
 
 	nnSite := sites[2]
-	require.Equal(t, "nn", nnSite.Language.Lang)
-	taxNn := nnSite.getPage(KindTaxonomyTerm, "lag")
+	require.Equal(t, "nn", nnSite.language.Lang)
+	taxNn := nnSite.getPage(page.KindTaxonomyTerm, "lag")
 	require.NotNil(t, taxNn)
 	require.Len(t, taxNn.Translations(), 1)
-	require.Equal(t, "nb", taxNn.Translations()[0].Lang())
+	require.Equal(t, "nb", taxNn.Translations()[0].Language().Lang)
 
-	taxTermNn := nnSite.getPage(KindTaxonomy, "lag", "sogndal")
+	taxTermNn := nnSite.getPage(page.KindTaxonomy, "lag", "sogndal")
 	require.NotNil(t, taxTermNn)
+	require.Equal(t, taxTermNn, nnSite.getPage(page.KindTaxonomy, "LAG", "SOGNDAL"))
 	require.Len(t, taxTermNn.Translations(), 1)
-	require.Equal(t, "nb", taxTermNn.Translations()[0].Lang())
+	require.Equal(t, "nb", taxTermNn.Translations()[0].Language().Lang)
 
 	// Check sitemap(s)
 	b.AssertFileContent("public/sitemap.xml",
@@ -371,59 +362,53 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) {
 	require.Len(t, enTags, 2, fmt.Sprintf("Tags in en: %v", enTags))
 	require.Len(t, frTags, 2, fmt.Sprintf("Tags in fr: %v", frTags))
 	require.NotNil(t, enTags["tag1"])
-	require.NotNil(t, frTags["frtag1"])
-	b.AssertFileContent("public/fr/plaques/frtag1/index.html", "Frtag1|Bonjour|http://example.com/blog/fr/plaques/frtag1/")
-	b.AssertFileContent("public/en/tags/tag1/index.html", "Tag1|Hello|http://example.com/blog/en/tags/tag1/")
+	require.NotNil(t, frTags["FRtag1"])
+	b.AssertFileContent("public/fr/plaques/FRtag1/index.html", "FRtag1|Bonjour|http://example.com/blog/fr/plaques/FRtag1/")
+	b.AssertFileContent("public/en/tags/tag1/index.html", "tag1|Hello|http://example.com/blog/en/tags/tag1/")
 
 	// Check Blackfriday config
-	require.True(t, strings.Contains(string(doc1fr.content()), "&laquo;"), string(doc1fr.content()))
-	require.False(t, strings.Contains(string(doc1en.content()), "&laquo;"), string(doc1en.content()))
-	require.True(t, strings.Contains(string(doc1en.content()), "&ldquo;"), string(doc1en.content()))
-
-	// Check that the drafts etc. are not built/processed/rendered.
-	assertShouldNotBuild(t, b.H)
+	require.True(t, strings.Contains(content(doc1fr), "&laquo;"), content(doc1fr))
+	require.False(t, strings.Contains(content(doc1en), "&laquo;"), content(doc1en))
+	require.True(t, strings.Contains(content(doc1en), "&ldquo;"), content(doc1en))
 
 	// en and nn have custom site menus
-	require.Len(t, frSite.Menus, 0, "fr: "+configSuffix)
-	require.Len(t, enSite.Menus, 1, "en: "+configSuffix)
-	require.Len(t, nnSite.Menus, 1, "nn: "+configSuffix)
+	require.Len(t, frSite.Menus(), 0, "fr: "+configSuffix)
+	require.Len(t, enSite.Menus(), 1, "en: "+configSuffix)
+	require.Len(t, nnSite.Menus(), 1, "nn: "+configSuffix)
 
-	require.Equal(t, "Home", enSite.Menus["main"].ByName()[0].Name)
-	require.Equal(t, "Heim", nnSite.Menus["main"].ByName()[0].Name)
-
-	// Issue #1302
-	require.Equal(t, template.URL(""), enSite.RegularPages[0].RSSLink())
+	require.Equal(t, "Home", enSite.Menus()["main"].ByName()[0].Name)
+	require.Equal(t, "Heim", nnSite.Menus()["main"].ByName()[0].Name)
 
 	// Issue #3108
-	prevPage := enSite.RegularPages[0].PrevPage
+	prevPage := enSite.RegularPages()[0].Prev()
 	require.NotNil(t, prevPage)
-	require.Equal(t, KindPage, prevPage.Kind)
+	require.Equal(t, page.KindPage, prevPage.Kind())
 
 	for {
 		if prevPage == nil {
 			break
 		}
-		require.Equal(t, KindPage, prevPage.Kind)
-		prevPage = prevPage.PrevPage
+		require.Equal(t, page.KindPage, prevPage.Kind())
+		prevPage = prevPage.Prev()
 	}
 
 	// Check bundles
-	bundleFr := frSite.getPage(KindPage, "bundles/b1/index.md")
+	b.AssertFileContent("public/fr/bundles/b1/index.html", "RelPermalink: /blog/fr/bundles/b1/|")
+	bundleFr := frSite.getPage(page.KindPage, "bundles/b1/index.md")
 	require.NotNil(t, bundleFr)
-	require.Equal(t, "/blog/fr/bundles/b1/", bundleFr.RelPermalink())
-	require.Equal(t, 1, len(bundleFr.Resources))
-	logoFr := bundleFr.Resources.GetMatch("logo*")
+	require.Equal(t, 1, len(bundleFr.Resources()))
+	logoFr := bundleFr.Resources().GetMatch("logo*")
 	require.NotNil(t, logoFr)
-	require.Equal(t, "/blog/fr/bundles/b1/logo.png", logoFr.RelPermalink())
+	b.AssertFileContent("public/fr/bundles/b1/index.html", "Resources: image/png: /blog/fr/bundles/b1/logo.png")
 	b.AssertFileContent("public/fr/bundles/b1/logo.png", "PNG Data")
 
-	bundleEn := enSite.getPage(KindPage, "bundles/b1/index.en.md")
+	bundleEn := enSite.getPage(page.KindPage, "bundles/b1/index.en.md")
 	require.NotNil(t, bundleEn)
-	require.Equal(t, "/blog/en/bundles/b1/", bundleEn.RelPermalink())
-	require.Equal(t, 1, len(bundleEn.Resources))
-	logoEn := bundleEn.Resources.GetMatch("logo*")
+	b.AssertFileContent("public/en/bundles/b1/index.html", "RelPermalink: /blog/en/bundles/b1/|")
+	require.Equal(t, 1, len(bundleEn.Resources()))
+	logoEn := bundleEn.Resources().GetMatch("logo*")
 	require.NotNil(t, logoEn)
-	require.Equal(t, "/blog/en/bundles/b1/logo.png", logoEn.RelPermalink())
+	b.AssertFileContent("public/en/bundles/b1/index.html", "Resources: image/png: /blog/en/bundles/b1/logo.png")
 	b.AssertFileContent("public/en/bundles/b1/logo.png", "PNG Data")
 
 }
@@ -442,13 +427,13 @@ func TestMultiSitesRebuild(t *testing.T) {
 	sites := b.H.Sites
 	fs := b.Fs
 
-	b.AssertFileContent("public/en/sect/doc2/index.html", "Single: doc2|Hello|en|\n\n<h1 id=\"doc2\">doc2</h1>\n\n<p><em>some content</em>")
+	b.AssertFileContent("public/en/sect/doc2/index.html", "Single: doc2|Hello|en|", "\n\n<h1 id=\"doc2\">doc2</h1>\n\n<p><em>some content</em>")
 
 	enSite := sites[0]
 	frSite := sites[1]
 
-	assert.Len(enSite.RegularPages, 5)
-	assert.Len(frSite.RegularPages, 4)
+	assert.Len(enSite.RegularPages(), 5)
+	assert.Len(frSite.RegularPages(), 4)
 
 	// Verify translations
 	b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Hello")
@@ -458,6 +443,10 @@ func TestMultiSitesRebuild(t *testing.T) {
 	b.AssertFileContent("public/fr/sect/doc1/index.html", "Single", "Shortcode: Bonjour")
 	b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Single", "Shortcode: Hello")
 
+	homeEn := enSite.getPage(page.KindHome)
+	require.NotNil(t, homeEn)
+	assert.Len(homeEn.Translations(), 3)
+
 	contentFs := b.H.BaseFs.Content.Fs
 
 	for i, this := range []struct {
@@ -478,15 +467,15 @@ func TestMultiSitesRebuild(t *testing.T) {
 			},
 			[]fsnotify.Event{{Name: filepath.FromSlash("content/sect/doc2.en.md"), Op: fsnotify.Remove}},
 			func(t *testing.T) {
-				assert.Len(enSite.RegularPages, 4, "1 en removed")
+				assert.Len(enSite.RegularPages(), 4, "1 en removed")
 
 				// Check build stats
-				require.Equal(t, 1, enSite.draftCount, "Draft")
-				require.Equal(t, 1, enSite.futureCount, "Future")
-				require.Equal(t, 1, enSite.expiredCount, "Expired")
-				require.Equal(t, 0, frSite.draftCount, "Draft")
-				require.Equal(t, 1, frSite.futureCount, "Future")
-				require.Equal(t, 1, frSite.expiredCount, "Expired")
+				require.Equal(t, 1, enSite.buildStats.draftCount, "Draft")
+				require.Equal(t, 1, enSite.buildStats.futureCount, "Future")
+				require.Equal(t, 1, enSite.buildStats.expiredCount, "Expired")
+				require.Equal(t, 0, frSite.buildStats.draftCount, "Draft")
+				require.Equal(t, 1, frSite.buildStats.futureCount, "Future")
+				require.Equal(t, 1, frSite.buildStats.expiredCount, "Expired")
 			},
 		},
 		{
@@ -501,12 +490,12 @@ func TestMultiSitesRebuild(t *testing.T) {
 				{Name: filepath.FromSlash("content/new1.fr.md"), Op: fsnotify.Create},
 			},
 			func(t *testing.T) {
-				assert.Len(enSite.RegularPages, 6)
-				assert.Len(enSite.AllPages, 34)
-				assert.Len(frSite.RegularPages, 5)
-				require.Equal(t, "new_fr_1", frSite.RegularPages[3].title)
-				require.Equal(t, "new_en_2", enSite.RegularPages[0].title)
-				require.Equal(t, "new_en_1", enSite.RegularPages[1].title)
+				assert.Len(enSite.RegularPages(), 6)
+				assert.Len(enSite.AllPages(), 34)
+				assert.Len(frSite.RegularPages(), 5)
+				require.Equal(t, "new_fr_1", frSite.RegularPages()[3].Title())
+				require.Equal(t, "new_en_2", enSite.RegularPages()[0].Title())
+				require.Equal(t, "new_en_1", enSite.RegularPages()[1].Title())
 
 				rendered := readDestination(t, fs, "public/en/new1/index.html")
 				require.True(t, strings.Contains(rendered, "new_en_1"), rendered)
@@ -521,7 +510,7 @@ func TestMultiSitesRebuild(t *testing.T) {
 			},
 			[]fsnotify.Event{{Name: filepath.FromSlash("content/sect/doc1.en.md"), Op: fsnotify.Write}},
 			func(t *testing.T) {
-				assert.Len(enSite.RegularPages, 6)
+				assert.Len(enSite.RegularPages(), 6)
 				doc1 := readDestination(t, fs, "public/en/sect/doc1-slug/index.html")
 				require.True(t, strings.Contains(doc1, "CHANGED"), doc1)
 
@@ -539,8 +528,8 @@ func TestMultiSitesRebuild(t *testing.T) {
 				{Name: filepath.FromSlash("content/new1.en.md"), Op: fsnotify.Rename},
 			},
 			func(t *testing.T) {
-				assert.Len(enSite.RegularPages, 6, "Rename")
-				require.Equal(t, "new_en_1", enSite.RegularPages[1].title)
+				assert.Len(enSite.RegularPages(), 6, "Rename")
+				require.Equal(t, "new_en_1", enSite.RegularPages()[1].Title())
 				rendered := readDestination(t, fs, "public/en/new1renamed/index.html")
 				require.True(t, strings.Contains(rendered, "new_en_1"), rendered)
 			}},
@@ -554,9 +543,9 @@ func TestMultiSitesRebuild(t *testing.T) {
 			},
 			[]fsnotify.Event{{Name: filepath.FromSlash("layouts/_default/single.html"), Op: fsnotify.Write}},
 			func(t *testing.T) {
-				assert.Len(enSite.RegularPages, 6)
-				assert.Len(enSite.AllPages, 34)
-				assert.Len(frSite.RegularPages, 5)
+				assert.Len(enSite.RegularPages(), 6)
+				assert.Len(enSite.AllPages(), 34)
+				assert.Len(frSite.RegularPages(), 5)
 				doc1 := readDestination(t, fs, "public/en/sect/doc1-slug/index.html")
 				require.True(t, strings.Contains(doc1, "Template Changed"), doc1)
 			},
@@ -571,18 +560,18 @@ func TestMultiSitesRebuild(t *testing.T) {
 			},
 			[]fsnotify.Event{{Name: filepath.FromSlash("i18n/fr.yaml"), Op: fsnotify.Write}},
 			func(t *testing.T) {
-				assert.Len(enSite.RegularPages, 6)
-				assert.Len(enSite.AllPages, 34)
-				assert.Len(frSite.RegularPages, 5)
+				assert.Len(enSite.RegularPages(), 6)
+				assert.Len(enSite.AllPages(), 34)
+				assert.Len(frSite.RegularPages(), 5)
 				docEn := readDestination(t, fs, "public/en/sect/doc1-slug/index.html")
 				require.True(t, strings.Contains(docEn, "Hello"), "No Hello")
 				docFr := readDestination(t, fs, "public/fr/sect/doc1/index.html")
 				require.True(t, strings.Contains(docFr, "Salut"), "No Salut")
 
-				homeEn := enSite.getPage(KindHome)
+				homeEn := enSite.getPage(page.KindHome)
 				require.NotNil(t, homeEn)
 				assert.Len(homeEn.Translations(), 3)
-				require.Equal(t, "fr", homeEn.Translations()[0].Lang())
+				require.Equal(t, "fr", homeEn.Translations()[0].Language().Lang)
 
 			},
 		},
@@ -595,9 +584,9 @@ func TestMultiSitesRebuild(t *testing.T) {
 				{Name: filepath.FromSlash("layouts/shortcodes/shortcode.html"), Op: fsnotify.Write},
 			},
 			func(t *testing.T) {
-				assert.Len(enSite.RegularPages, 6)
-				assert.Len(enSite.AllPages, 34)
-				assert.Len(frSite.RegularPages, 5)
+				assert.Len(enSite.RegularPages(), 6)
+				assert.Len(enSite.AllPages(), 34)
+				assert.Len(frSite.RegularPages(), 5)
 				b.AssertFileContent("public/fr/sect/doc1/index.html", "Single", "Modified Shortcode: Salut")
 				b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Single", "Modified Shortcode: Hello")
 			},
@@ -617,23 +606,6 @@ func TestMultiSitesRebuild(t *testing.T) {
 		this.assertFunc(t)
 	}
 
-	// Check that the drafts etc. are not built/processed/rendered.
-	assertShouldNotBuild(t, b.H)
-
-}
-
-func assertShouldNotBuild(t *testing.T, sites *HugoSites) {
-	s := sites.Sites[0]
-
-	for _, p := range s.rawAllPages {
-		// No HTML when not processed
-		require.Equal(t, p.shouldBuild(), bytes.Contains(p.workContent, []byte("</")), p.BaseFileName()+": "+string(p.workContent))
-
-		require.Equal(t, p.shouldBuild(), p.content() != "", fmt.Sprintf("%v:%v", p.content(), p.shouldBuild()))
-
-		require.Equal(t, p.shouldBuild(), p.content() != "", p.BaseFileName())
-
-	}
 }
 
 func TestAddNewLanguage(t *testing.T) {
@@ -671,31 +643,32 @@ title = "Svenska"
 	enSite := sites.Sites[0]
 	svSite := sites.Sites[1]
 	frSite := sites.Sites[2]
-	require.True(t, enSite.Language.Lang == "en", enSite.Language.Lang)
-	require.True(t, svSite.Language.Lang == "sv", svSite.Language.Lang)
-	require.True(t, frSite.Language.Lang == "fr", frSite.Language.Lang)
+	require.True(t, enSite.language.Lang == "en", enSite.language.Lang)
+	require.True(t, svSite.language.Lang == "sv", svSite.language.Lang)
+	require.True(t, frSite.language.Lang == "fr", frSite.language.Lang)
 
-	homeEn := enSite.getPage(KindHome)
+	homeEn := enSite.getPage(page.KindHome)
 	require.NotNil(t, homeEn)
 	require.Len(t, homeEn.Translations(), 4)
-	require.Equal(t, "sv", homeEn.Translations()[0].Lang())
 
-	require.Len(t, enSite.RegularPages, 5)
-	require.Len(t, frSite.RegularPages, 4)
+	require.Equal(t, "sv", homeEn.Translations()[0].Language().Lang)
+
+	require.Len(t, enSite.RegularPages(), 5)
+	require.Len(t, frSite.RegularPages(), 4)
 
 	// Veriy Swedish site
-	require.Len(t, svSite.RegularPages, 1)
-	svPage := svSite.RegularPages[0]
+	require.Len(t, svSite.RegularPages(), 1)
+	svPage := svSite.RegularPages()[0]
 
-	require.Equal(t, "Swedish Contentfile", svPage.title)
-	require.Equal(t, "sv", svPage.Lang())
+	require.Equal(t, "Swedish Contentfile", svPage.Title())
+	require.Equal(t, "sv", svPage.Language().Lang)
 	require.Len(t, svPage.Translations(), 2)
 	require.Len(t, svPage.AllTranslations(), 3)
-	require.Equal(t, "en", svPage.Translations()[0].Lang())
+	require.Equal(t, "en", svPage.Translations()[0].Language().Lang)
 
 	// Regular pages have no children
-	require.Len(t, svPage.Pages, 0)
-	require.Len(t, svPage.data["Pages"], 0)
+	require.Len(t, svPage.Pages(), 0)
+	require.Len(t, svPage.Data().(page.Data).Pages(), 0)
 
 }
 
@@ -782,12 +755,12 @@ Some text. Some more text.
 	content = append(content, []string{"s2/_index.md", fmt.Sprintf(contentTempl, defaultOutputs, fmt.Sprintf("S %d", 2), 2, true)}...)
 
 	b.WithSimpleConfigFile()
-	b.WithTemplates("layouts/_default/single.html", `Single: {{ .Content }}`)
+	b.WithTemplates("layouts/_default/single.html", `Single: {{ .Content }}|RelPermalink: {{ .RelPermalink }}|Permalink: {{ .Permalink }}`)
 	b.WithTemplates("layouts/_default/myview.html", `View: {{ len .Content }}`)
-	b.WithTemplates("layouts/_default/single.json", `Single JSON: {{ .Content }}`)
+	b.WithTemplates("layouts/_default/single.json", `Single JSON: {{ .Content }}|RelPermalink: {{ .RelPermalink }}|Permalink: {{ .Permalink }}`)
 	b.WithTemplates("layouts/_default/list.html", `
 Page: {{ .Paginator.PageNumber }}
-P: {{ path.Join .Path }}
+P: {{ with .File }}{{ path.Join .Path }}{{ end }}
 List: {{ len .Paginator.Pages }}|List Content: {{ len .Content }}
 {{ $shuffled :=  where .Site.RegularPages "Params.multioutput" true | shuffle }}
 {{ $first5 := $shuffled | first 5 }}
@@ -810,7 +783,7 @@ END
 			if i%10 == 0 {
 				section = "s2"
 			}
-			checkContent(b, fmt.Sprintf("public/%s/page%d/index.html", section, i), 8343, contentMatchers...)
+			checkContent(b, fmt.Sprintf("public/%s/page%d/index.html", section, i), contentMatchers...)
 		}
 	}
 
@@ -819,48 +792,158 @@ END
 		if i%10 == 0 {
 			section = "s2"
 		}
-		checkContent(b, fmt.Sprintf("public/%s/page%d/index.json", section, i), 8348, contentMatchers...)
+		checkContent(b, fmt.Sprintf("public/%s/page%d/index.json", section, i), contentMatchers...)
 	}
 
-	checkContent(b, "public/s1/index.html", 184, "P: s1/_index.md\nList: 10|List Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335\n\nRender 1: View: 8335\n\nRender 2: View: 8335\n\nRender 3: View: 8335\n\nRender 4: View: 8335\n\nEND\n")
-	checkContent(b, "public/s2/index.html", 184, "P: s2/_index.md\nList: 10|List Content: 8335", "Render 4: View: 8335\n\nEND")
-	checkContent(b, "public/index.html", 181, "P: _index.md\nList: 10|List Content: 8335", "4: View: 8335\n\nEND")
+	checkContent(b, "public/s1/index.html", "P: s1/_index.md\nList: 10|List Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335\n\nRender 1: View: 8335\n\nRender 2: View: 8335\n\nRender 3: View: 8335\n\nRender 4: View: 8335\n\nEND\n")
+	checkContent(b, "public/s2/index.html", "P: s2/_index.md\nList: 10|List Content: 8335", "Render 4: View: 8335\n\nEND")
+	checkContent(b, "public/index.html", "P: _index.md\nList: 10|List Content: 8335", "4: View: 8335\n\nEND")
 
-	// Chek paginated pages
+	// Check paginated pages
 	for i := 2; i <= 9; i++ {
-		checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), 181, fmt.Sprintf("Page: %d", i), "Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335", "Render 4: View: 8335\n\nEND")
+		checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), fmt.Sprintf("Page: %d", i), "Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335", "Render 4: View: 8335\n\nEND")
 	}
 }
 
-func checkContent(s *sitesBuilder, filename string, length int, matches ...string) {
+func checkContent(s *sitesBuilder, filename string, matches ...string) {
 	content := readDestination(s.T, s.Fs, filename)
 	for _, match := range matches {
 		if !strings.Contains(content, match) {
 			s.Fatalf("No match for %q in content for %s\n%q", match, filename, content)
 		}
 	}
-	if len(content) != length {
-		s.Fatalf("got %d expected %d", len(content), length)
+
+}
+
+func TestTranslationsFromContentToNonContent(t *testing.T) {
+	b := newTestSitesBuilder(t)
+	b.WithConfigFile("toml", `
+
+baseURL = "http://example.com/"
+
+defaultContentLanguage = "en"
+
+[languages]
+[languages.en]
+weight = 10
+contentDir = "content/en"
+[languages.nn]
+weight = 20
+contentDir = "content/nn"
+
+
+`)
+
+	b.WithContent("en/mysection/_index.md", `
+---
+Title: My Section
+---
+
+`)
+
+	b.WithContent("en/_index.md", `
+---
+Title: My Home
+---
+
+`)
+
+	b.WithContent("en/categories/mycat/_index.md", `
+---
+Title: My MyCat
+---
+
+`)
+
+	b.WithContent("en/categories/_index.md", `
+---
+Title: My categories
+---
+
+`)
+
+	for _, lang := range []string{"en", "nn"} {
+
+		b.WithContent(lang+"/mysection/page.md", `
+---
+Title: My Page
+categories: ["mycat"]
+---
+
+`)
+
+	}
+
+	b.Build(BuildCfg{})
+
+	for _, path := range []string{
+		"/",
+		"/mysection",
+		"/categories",
+		"/categories/mycat",
+	} {
+
+		t.Run(path, func(t *testing.T) {
+			assert := require.New(t)
+
+			s1, _ := b.H.Sites[0].getPageNew(nil, path)
+			s2, _ := b.H.Sites[1].getPageNew(nil, path)
+
+			assert.NotNil(s1)
+			assert.NotNil(s2)
+
+			assert.Equal(1, len(s1.Translations()))
+			assert.Equal(1, len(s2.Translations()))
+			assert.Equal(s2, s1.Translations()[0])
+			assert.Equal(s1, s2.Translations()[0])
+
+			m1 := s1.Translations().MergeByLanguage(s2.Translations())
+			m2 := s2.Translations().MergeByLanguage(s1.Translations())
+
+			assert.Equal(1, len(m1))
+			assert.Equal(1, len(m2))
+		})
+
 	}
 }
 
+// https://github.com/gohugoio/hugo/issues/5777
 func TestTableOfContentsInShortcodes(t *testing.T) {
 	t.Parallel()
 
 	b := newMultiSiteTestDefaultBuilder(t)
 
 	b.WithTemplatesAdded("layouts/shortcodes/toc.html", tocShortcode)
+	b.WithTemplatesAdded("layouts/shortcodes/wrapper.html", "{{ .Inner }}")
 	b.WithContent("post/simple.en.md", tocPageSimple)
+	b.WithContent("post/variants1.en.md", tocPageVariants1)
+	b.WithContent("post/variants2.en.md", tocPageVariants2)
+
 	b.WithContent("post/withSCInHeading.en.md", tocPageWithShortcodesInHeadings)
 
 	b.CreateSites().Build(BuildCfg{})
 
-	b.AssertFileContent("public/en/post/simple/index.html", tocPageSimpleExpected)
+	b.AssertFileContent("public/en/post/simple/index.html",
+		tocPageSimpleExpected,
+		// Make sure it is inserted twice
+		`TOC1: <nav id="TableOfContents">`,
+		`TOC2: <nav id="TableOfContents">`,
+	)
+
+	b.AssertFileContentFn("public/en/post/variants1/index.html", func(s string) bool {
+		return strings.Count(s, "TableOfContents") == 4
+	})
+	b.AssertFileContentFn("public/en/post/variants2/index.html", func(s string) bool {
+		return strings.Count(s, "TableOfContents") == 6
+	})
+
 	b.AssertFileContent("public/en/post/withSCInHeading/index.html", tocPageWithShortcodesInHeadingsExpected)
 }
 
 var tocShortcode = `
-{{ .Page.TableOfContents }}
+TOC1: {{ .Page.TableOfContents }}
+
+TOC2: {{ .Page.TableOfContents }}
 `
 
 func TestSelfReferencedContentInShortcode(t *testing.T) {
@@ -901,6 +984,41 @@ Even more text.
 Lorem ipsum...
 `
 
+var tocPageVariants1 = `---
+title: tocTest
+publishdate: "2000-01-01"
+---
+Variant 1:
+{{% wrapper %}}
+{{< toc >}}
+{{% /wrapper %}}
+# Heading 1
+
+Variant 3:
+{{% toc %}}
+
+`
+
+var tocPageVariants2 = `---
+title: tocTest
+publishdate: "2000-01-01"
+---
+Variant 1:
+{{% wrapper %}}
+{{< toc >}}
+{{% /wrapper %}}
+# Heading 1
+
+Variant 2:
+{{< wrapper >}}
+{{< toc >}}
+{{< /wrapper >}}
+
+Variant 3:
+{{% toc %}}
+
+`
+
 var tocPageSimpleExpected = `<nav id="TableOfContents">
 <ul>
 <li><a href="#1">Heading 1</a>
@@ -958,6 +1076,7 @@ paginate = 1
 disablePathToLower = true
 defaultContentLanguage = "{{ .DefaultContentLanguage }}"
 defaultContentLanguageInSubdir = {{ .DefaultContentLanguageInSubdir }}
+enableRobotsTXT = true
 
 [permalinks]
 other = "/somewhere/else/:filename"
@@ -1015,6 +1134,7 @@ disablePathToLower: true
 paginate: 1
 defaultContentLanguage: "{{ .DefaultContentLanguage }}"
 defaultContentLanguageInSubdir: {{ .DefaultContentLanguageInSubdir }}
+enableRobotsTXT: true
 
 permalinks:
     other: "/somewhere/else/:filename"
@@ -1073,6 +1193,7 @@ var multiSiteJSONConfigTemplate = `
   "disablePathToLower": true,
   "defaultContentLanguage": "{{ .DefaultContentLanguage }}",
   "defaultContentLanguageInSubdir": true,
+  "enableRobotsTXT": true,
   "permalinks": {
     "other": "/somewhere/else/:filename"
   },
@@ -1170,7 +1291,23 @@ func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string {
 	b, err := afero.ReadFile(fs, filename)
 	if err != nil {
 		// Print some debug info
-		root := strings.Split(filename, helpers.FilePathSeparator)[0]
+		hadSlash := strings.HasPrefix(filename, helpers.FilePathSeparator)
+		start := 0
+		if hadSlash {
+			start = 1
+		}
+		end := start + 1
+
+		parts := strings.Split(filename, helpers.FilePathSeparator)
+		if parts[start] == "work" {
+			end++
+		}
+
+		root := filepath.Join(parts[start:end]...)
+		if hadSlash {
+			root = helpers.FilePathSeparator + root
+		}
+
 		helpers.PrintFs(fs, root, os.Stdout)
 		Fatalf(t, "Failed to read file: %s", err)
 	}
@@ -1262,8 +1399,8 @@ NOTE: slug should be used as URL
 title: doc1
 weight: 1
 plaques:
- - frtag1
- - frtag2
+ - FRtag1
+ - FRtag2
 publishdate: "2000-01-04"
 ---
 # doc1
@@ -1293,7 +1430,7 @@ aliases: [/en/al/alias1,/al/alias2/]
 tags:
  - tag2
  - tag1
-url: /superbob
+url: /superbob/
 ---
 # doc3
 *some content*
@@ -1303,7 +1440,7 @@ NOTE: third 'en' doc, should trigger pagination on home page.
 title: doc4
 weight: 4
 plaques:
- - frtag1
+ - FRtag1
 publishdate: "2000-01-05"
 ---
 # doc4