Allow headless bundles to list pages via $page.Pages and $page.RegularPages

Fixes #7075

10 files changed, 288 insertions, 72 deletions

diff --git a/docs/content/en/content-management/build-options.md b/docs/content/en/content-management/build-options.md
index f402b9014..47f39747d 100644
--- a/docs/content/en/content-management/build-options.md
+++ b/docs/content/en/content-management/build-options.md
@@ -21,7 +21,7 @@ They are stored in a reserved Front Matter object named `_build` with the follow
 ```yaml
 _build:
   render: true
-  list: true
+  list: always
   publishResources: true
 ```
 
@@ -29,6 +29,20 @@ _build:
 If true, the page will be treated as a published page, holding its dedicated output files (`index.html`, etc...) and permalink.
 
 #### list
+
+Note that we extended this property from a boolean to an enum in Hugo 0.58.0.
+
+Valid values are:
+
+never
+: The page will not be incldued in any page collection.
+
+always (default)
+: The page will be included in all page collections, e.g. `site.RegularPages`, `$page.Pages`.
+
+local
+: The page will be included in any _local_ page collection, e.g. `$page.RegularPages`, `$page.Pages`. One use case for this would be to create fully navigable, but headless content sections. {{< new-in "0.58.0" >}}
+
 If true, the page will be treated as part of the project's collections and, when appropriate, returned by Hugo's listing methods (`.Pages`, `.RegularPages` etc...).
 
 #### publishResources
diff --git a/hugolib/content_map.go b/hugolib/content_map.go
index 2ea36ff31..7b28965ac 100644
--- a/hugolib/content_map.go
+++ b/hugolib/content_map.go
@@ -789,6 +789,12 @@ func (t contentTrees) DeletePrefix(prefix string) int {
 
 type contentTreeNodeCallback func(s string, n *contentNode) bool
 
+func newContentTreeFilter(fn func(n *contentNode) bool) contentTreeNodeCallback {
+	return func(s string, n *contentNode) bool {
+		return fn(n)
+	}
+}
+
 var (
 	contentTreeNoListFilter = func(s string, n *contentNode) bool {
 		if n.p == nil {
@@ -805,43 +811,36 @@ var (
 	}
 )
 
-func (c *contentTree) WalkPrefixListable(prefix string, fn contentTreeNodeCallback) {
-	c.WalkPrefixFilter(prefix, contentTreeNoListFilter, fn)
-}
-
-func (c *contentTree) WalkPrefixFilter(prefix string, filter, walkFn contentTreeNodeCallback) {
-	c.WalkPrefix(prefix, func(s string, v interface{}) bool {
-		n := v.(*contentNode)
-		if filter(s, n) {
-			return false
-		}
-		return walkFn(s, n)
-	})
-}
+func (c *contentTree) WalkQuery(query pageMapQuery, walkFn contentTreeNodeCallback) {
+	filter := query.Filter
+	if filter == nil {
+		filter = contentTreeNoListFilter
+	}
+	if query.Prefix != "" {
+		c.WalkPrefix(query.Prefix, func(s string, v interface{}) bool {
+			n := v.(*contentNode)
+			if filter != nil && filter(s, n) {
+				return false
+			}
+			return walkFn(s, n)
+		})
 
-func (c *contentTree) WalkListable(fn contentTreeNodeCallback) {
-	c.WalkFilter(contentTreeNoListFilter, fn)
-}
+		return
+	}
 
-func (c *contentTree) WalkFilter(filter, walkFn contentTreeNodeCallback) {
 	c.Walk(func(s string, v interface{}) bool {
 		n := v.(*contentNode)
-		if filter(s, n) {
+		if filter != nil && filter(s, n) {
 			return false
 		}
 		return walkFn(s, n)
 	})
 }
 
-func (c contentTrees) WalkListable(fn contentTreeNodeCallback) {
-	for _, tree := range c {
-		tree.WalkListable(fn)
-	}
-}
-
 func (c contentTrees) WalkRenderable(fn contentTreeNodeCallback) {
+	query := pageMapQuery{Filter: contentTreeNoRenderFilter}
 	for _, tree := range c {
-		tree.WalkFilter(contentTreeNoRenderFilter, fn)
+		tree.WalkQuery(query, fn)
 	}
 }
 
@@ -931,44 +930,73 @@ func (c *contentTreeRef) getSection() (string, *contentNode) {
 	return c.m.getSection(c.key)
 }
 
-func (c *contentTreeRef) collectPages() page.Pages {
+func (c *contentTreeRef) getPages() page.Pages {
 	var pas page.Pages
-	c.m.collectPages(c.key+cmBranchSeparator, func(c *contentNode) {
-		pas = append(pas, c.p)
-	})
+	c.m.collectPages(
+		pageMapQuery{
+			Prefix: c.key + cmBranchSeparator,
+			Filter: c.n.p.m.getListFilter(true),
+		},
+		func(c *contentNode) {
+			pas = append(pas, c.p)
+		},
+	)
 	page.SortByDefault(pas)
 
 	return pas
 }
 
-func (c *contentTreeRef) collectPagesRecursive() page.Pages {
+func (c *contentTreeRef) getPagesRecursive() page.Pages {
 	var pas page.Pages
-	c.m.collectPages(c.key+cmBranchSeparator, func(c *contentNode) {
+
+	query := pageMapQuery{
+		Filter: c.n.p.m.getListFilter(true),
+	}
+
+	query.Prefix = c.key + cmBranchSeparator
+	c.m.collectPages(query, func(c *contentNode) {
 		pas = append(pas, c.p)
 	})
-	c.m.collectPages(c.key+"/", func(c *contentNode) {
+
+	query.Prefix = c.key + "/"
+	c.m.collectPages(query, func(c *contentNode) {
 		pas = append(pas, c.p)
 	})
+
 	page.SortByDefault(pas)
 
 	return pas
 }
 
-func (c *contentTreeRef) collectPagesAndSections() page.Pages {
+func (c *contentTreeRef) getPagesAndSections() page.Pages {
 	var pas page.Pages
-	c.m.collectPagesAndSections(c.key, func(c *contentNode) {
+
+	query := pageMapQuery{
+		Filter: c.n.p.m.getListFilter(true),
+		Prefix: c.key,
+	}
+
+	c.m.collectPagesAndSections(query, func(c *contentNode) {
 		pas = append(pas, c.p)
 	})
+
 	page.SortByDefault(pas)
 
 	return pas
 }
 
-func (c *contentTreeRef) collectSections() page.Pages {
+func (c *contentTreeRef) getSections() page.Pages {
 	var pas page.Pages
-	c.m.collectSections(c.key, func(c *contentNode) {
+
+	query := pageMapQuery{
+		Filter: c.n.p.m.getListFilter(true),
+		Prefix: c.key,
+	}
+
+	c.m.collectSections(query, func(c *contentNode) {
 		pas = append(pas, c.p)
 	})
+
 	page.SortByDefault(pas)
 
 	return pas
diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go
index 4ba97f511..bcab9ffa9 100644
--- a/hugolib/content_map_page.go
+++ b/hugolib/content_map_page.go
@@ -606,36 +606,47 @@ func (m *pageMap) attachPageToViews(s string, b *contentNode) {
 	}
 }
 
-func (m *pageMap) collectPages(prefix string, fn func(c *contentNode)) error {
-	m.pages.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+type pageMapQuery struct {
+	Prefix string
+	Filter contentTreeNodeCallback
+}
+
+func (m *pageMap) collectPages(query pageMapQuery, fn func(c *contentNode)) error {
+	if query.Filter == nil {
+		query.Filter = contentTreeNoListFilter
+	}
+
+	m.pages.WalkQuery(query, func(s string, n *contentNode) bool {
 		fn(n)
 		return false
 	})
+
 	return nil
 }
 
-func (m *pageMap) collectPagesAndSections(prefix string, fn func(c *contentNode)) error {
-	if err := m.collectSections(prefix, fn); err != nil {
+func (m *pageMap) collectPagesAndSections(query pageMapQuery, fn func(c *contentNode)) error {
+	if err := m.collectSections(query, fn); err != nil {
 		return err
 	}
 
-	if err := m.collectPages(prefix+cmBranchSeparator, fn); err != nil {
+	query.Prefix = query.Prefix + cmBranchSeparator
+	if err := m.collectPages(query, fn); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (m *pageMap) collectSections(prefix string, fn func(c *contentNode)) error {
+func (m *pageMap) collectSections(query pageMapQuery, fn func(c *contentNode)) error {
 	var level int
-	isHome := prefix == "/"
+	isHome := query.Prefix == "/"
 
 	if !isHome {
-		level = strings.Count(prefix, "/")
+		level = strings.Count(query.Prefix, "/")
 	}
 
-	return m.collectSectionsFn(prefix, func(s string, c *contentNode) bool {
-		if s == prefix {
+	return m.collectSectionsFn(query, func(s string, c *contentNode) bool {
+		if s == query.Prefix {
 			return false
 		}
 
@@ -649,27 +660,28 @@ func (m *pageMap) collectSections(prefix string, fn func(c *contentNode)) error
 	})
 }
 
-func (m *pageMap) collectSectionsFn(prefix string, fn func(s string, c *contentNode) bool) error {
-	if !strings.HasSuffix(prefix, "/") {
-		prefix += "/"
+func (m *pageMap) collectSectionsFn(query pageMapQuery, fn func(s string, c *contentNode) bool) error {
+
+	if !strings.HasSuffix(query.Prefix, "/") {
+		query.Prefix += "/"
 	}
 
-	m.sections.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+	m.sections.WalkQuery(query, func(s string, n *contentNode) bool {
 		return fn(s, n)
 	})
 
 	return nil
 }
 
-func (m *pageMap) collectSectionsRecursiveIncludingSelf(prefix string, fn func(c *contentNode)) error {
-	return m.collectSectionsFn(prefix, func(s string, c *contentNode) bool {
+func (m *pageMap) collectSectionsRecursiveIncludingSelf(query pageMapQuery, fn func(c *contentNode)) error {
+	return m.collectSectionsFn(query, func(s string, c *contentNode) bool {
 		fn(c)
 		return false
 	})
 }
 
 func (m *pageMap) collectTaxonomies(prefix string, fn func(c *contentNode)) error {
-	m.taxonomies.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+	m.taxonomies.WalkQuery(pageMapQuery{Prefix: prefix}, func(s string, n *contentNode) bool {
 		fn(n)
 		return false
 	})
@@ -797,21 +809,21 @@ type pagesMapBucket struct {
 
 func (b *pagesMapBucket) getPages() page.Pages {
 	b.pagesInit.Do(func() {
-		b.pages = b.owner.treeRef.collectPages()
+		b.pages = b.owner.treeRef.getPages()
 		page.SortByDefault(b.pages)
 	})
 	return b.pages
 }
 
 func (b *pagesMapBucket) getPagesRecursive() page.Pages {
-	pages := b.owner.treeRef.collectPagesRecursive()
+	pages := b.owner.treeRef.getPagesRecursive()
 	page.SortByDefault(pages)
 	return pages
 }
 
 func (b *pagesMapBucket) getPagesAndSections() page.Pages {
 	b.pagesAndSectionsInit.Do(func() {
-		b.pagesAndSections = b.owner.treeRef.collectPagesAndSections()
+		b.pagesAndSections = b.owner.treeRef.getPagesAndSections()
 	})
 	return b.pagesAndSections
 }
@@ -821,7 +833,7 @@ func (b *pagesMapBucket) getSections() page.Pages {
 		if b.owner.treeRef == nil {
 			return
 		}
-		b.sections = b.owner.treeRef.collectSections()
+		b.sections = b.owner.treeRef.getSections()
 	})
 
 	return b.sections
diff --git a/hugolib/disableKinds_test.go b/hugolib/disableKinds_test.go
index b1c00b48e..9ac30442e 100644
--- a/hugolib/disableKinds_test.go
+++ b/hugolib/disableKinds_test.go
@@ -66,7 +66,26 @@ title: Headless
 headless: true
 ---
 
-`)
+
+`, "headless-local/_index.md", `
+---
+title: Headless Local Lists
+cascade:
+    _build:
+        render: false
+        list: local
+        publishResources: false
+---
+
+`, "headless-local/headless-local-page.md", "---\ntitle: Headless Local Page\n---",
+			"headless-local/sub/_index.md", `
+---
+title: Headless Local Lists Sub
+---
+
+`, "headless-local/sub/headless-local-sub-page.md", "---\ntitle: Headless Local Sub Page\n---",
+		)
+
 		b.WithSourceFile("content/sect/headlessbundle/data.json", "DATA")
 		b.WithSourceFile("content/sect/no-publishresources/data.json", "DATA")
 
@@ -93,8 +112,11 @@ headless: true
 		return nil
 	}
 
-	getPageInPagePages := func(p page.Page, ref string) page.Page {
-		for _, pages := range []page.Pages{p.Pages(), p.RegularPages(), p.Sections()} {
+	getPageInPagePages := func(p page.Page, ref string, pageCollections ...page.Pages) page.Page {
+		if len(pageCollections) == 0 {
+			pageCollections = []page.Pages{p.Pages(), p.RegularPages(), p.RegularPagesRecursive(), p.Sections()}
+		}
+		for _, pages := range pageCollections {
 			for _, p := range pages {
 				if ref == p.(*pageState).sourceRef() {
 					return p
@@ -240,6 +262,33 @@ headless: true
 
 	})
 
+	c.Run("Build config, local list", func(c *qt.C) {
+		b := newSitesBuilder(c, disableKind)
+		b.Build(BuildCfg{})
+		ref := "/headless-local"
+		sect := getPage(b, ref)
+		b.Assert(sect, qt.Not(qt.IsNil))
+		b.Assert(getPageInSitePages(b, ref), qt.IsNil)
+		b.Assert(getPageInSitePages(b, ref+"/headless-local-page"), qt.IsNil)
+		for i, p := range sect.RegularPages() {
+			fmt.Println("REG", i, p.(*pageState).sourceRef())
+		}
+
+		localPageRef := ref + "/headless-local-page.md"
+
+		b.Assert(getPageInPagePages(sect, localPageRef, sect.RegularPages()), qt.Not(qt.IsNil))
+		b.Assert(getPageInPagePages(sect, localPageRef, sect.RegularPagesRecursive()), qt.Not(qt.IsNil))
+		b.Assert(getPageInPagePages(sect, localPageRef, sect.Pages()), qt.Not(qt.IsNil))
+
+		ref = "/headless-local/sub"
+
+		sect = getPage(b, ref)
+		b.Assert(sect, qt.Not(qt.IsNil))
+
+		localPageRef = ref + "/headless-local-sub-page.md"
+		b.Assert(getPageInPagePages(sect, localPageRef), qt.Not(qt.IsNil))
+	})
+
 	c.Run("Build config, no render", func(c *qt.C) {
 		b := newSitesBuilder(c, disableKind)
 		b.Build(BuildCfg{})
diff --git a/hugolib/page.go b/hugolib/page.go
index f71c4d9b8..fddc25fa0 100644
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -147,7 +147,7 @@ func (p *pageState) GetTerms(taxonomy string) page.Pages {
 
 	var pas page.Pages
 
-	m.taxonomies.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+	m.taxonomies.WalkQuery(pageMapQuery{Prefix: prefix}, func(s string, n *contentNode) bool {
 		if _, found := m.taxonomyEntries.Get(s + self); found {
 			pas = append(pas, n.p)
 		}
diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go
index 7eb7cbfe1..87e955103 100644
--- a/hugolib/page__meta.go
+++ b/hugolib/page__meta.go
@@ -460,7 +460,7 @@ func (pm *pageMeta) setMetadata(parentBucket *pagesMapBucket, p *pageState, fron
 			isHeadless := cast.ToBool(v)
 			pm.params[loki] = isHeadless
 			if p.File().TranslationBaseName() == "index" && isHeadless {
-				pm.buildConfig.List = false
+				pm.buildConfig.List = pagemeta.Never
 				pm.buildConfig.Render = false
 			}
 		case "outputs":
@@ -613,7 +613,28 @@ func (pm *pageMeta) setMetadata(parentBucket *pagesMapBucket, p *pageState, fron
 }
 
 func (p *pageMeta) noList() bool {
-	return !p.buildConfig.List
+	return !p.buildConfig.ShouldList()
+}
+
+func (p *pageMeta) getListFilter(local bool) contentTreeNodeCallback {
+
+	return newContentTreeFilter(func(n *contentNode) bool {
+		if n == nil {
+			return true
+		}
+
+		var shouldList bool
+		switch n.p.m.buildConfig.List {
+		case pagemeta.Always:
+			shouldList = true
+		case pagemeta.Never:
+			shouldList = false
+		case pagemeta.ListLocally:
+			shouldList = local
+		}
+
+		return !shouldList
+	})
 }
 
 func (p *pageMeta) noRender() bool {
diff --git a/hugolib/site.go b/hugolib/site.go
index f85dc47e2..56fa654db 100644
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -248,7 +248,7 @@ func (s *Site) prepareInits() {
 	s.init.prevNextInSection = init.Branch(func() (interface{}, error) {
 
 		var sections page.Pages
-		s.home.treeRef.m.collectSectionsRecursiveIncludingSelf(s.home.treeRef.key, func(n *contentNode) {
+		s.home.treeRef.m.collectSectionsRecursiveIncludingSelf(pageMapQuery{Prefix: s.home.treeRef.key}, func(n *contentNode) {
 			sections = append(sections, n.p)
 		})
 
@@ -281,7 +281,7 @@ func (s *Site) prepareInits() {
 			treeRef := sect.(treeRefProvider).getTreeRef()
 
 			var pas page.Pages
-			treeRef.m.collectPages(treeRef.key+cmBranchSeparator, func(c *contentNode) {
+			treeRef.m.collectPages(pageMapQuery{Prefix: treeRef.key + cmBranchSeparator}, func(c *contentNode) {
 				pas = append(pas, c.p)
 			})
 			page.SortByDefault(pas)
@@ -293,7 +293,7 @@ func (s *Site) prepareInits() {
 		treeRef := s.home.getTreeRef()
 
 		var pas page.Pages
-		treeRef.m.collectPages(treeRef.key+cmBranchSeparator, func(c *contentNode) {
+		treeRef.m.collectPages(pageMapQuery{Prefix: treeRef.key + cmBranchSeparator}, func(c *contentNode) {
 			pas = append(pas, c.p)
 		})
 		page.SortByDefault(pas)
diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go
index 8ecff189a..2815c6d99 100644
--- a/hugolib/testhelpers_test.go
+++ b/hugolib/testhelpers_test.go
@@ -1021,7 +1021,7 @@ func printStringIndexes(s string) {
 }
 
 func isCI() bool {
-	return os.Getenv("CI") != "" && os.Getenv("CIRCLE_BRANCH") == ""
+	return (os.Getenv("CI") != "" || os.Getenv("CI_LOCAL") != "") && os.Getenv("CIRCLE_BRANCH") == ""
 }
 
 // See https://github.com/golang/go/issues/19280
diff --git a/resources/page/pagemeta/pagemeta.go b/resources/page/pagemeta/pagemeta.go
index 1f8afdc18..632f46df7 100644
--- a/resources/page/pagemeta/pagemeta.go
+++ b/resources/page/pagemeta/pagemeta.go
@@ -24,8 +24,14 @@ type URLPath struct {
 	Section   string
 }
 
+const (
+	Never       = "never"
+	Always      = "always"
+	ListLocally = "local"
+)
+
 var defaultBuildConfig = BuildConfig{
-	List:             true,
+	List:             Always,
 	Render:           true,
 	PublishResources: true,
 	set:              true,
@@ -35,8 +41,12 @@ var defaultBuildConfig = BuildConfig{
 // build process.
 type BuildConfig struct {
 	// Whether to add it to any of the page collections.
-	// Note that the page can still be found with .Site.GetPage.
-	List bool
+	// Note that the page can always be found with .Site.GetPage.
+	// Valid values: never, always, local.
+	// Setting it to 'local' means they will be available via the local
+	// page collections, e.g. $section.Pages.
+	// Note: before 0.57.2 this was a bool, so we accept those too.
+	List string
 
 	// Whether to render it.
 	Render bool
@@ -51,7 +61,7 @@ type BuildConfig struct {
 
 // Disable sets all options to their off value.
 func (b *BuildConfig) Disable() {
-	b.List = false
+	b.List = Never
 	b.Render = false
 	b.PublishResources = false
 	b.set = true
@@ -61,11 +71,29 @@ func (b BuildConfig) IsZero() bool {
 	return !b.set
 }
 
+func (b *BuildConfig) ShouldList() bool {
+	return b.List == Always || b.List == ListLocally
+}
+
 func DecodeBuildConfig(m interface{}) (BuildConfig, error) {
 	b := defaultBuildConfig
 	if m == nil {
 		return b, nil
 	}
+
 	err := mapstructure.WeakDecode(m, &b)
+
+	// In 0.67.1 we changed the list attribute from a bool to a string (enum).
+	// Bool values will become 0 or 1.
+	switch b.List {
+	case "0":
+		b.List = Never
+	case "1":
+		b.List = Always
+	case Always, Never, ListLocally:
+	default:
+		b.List = Always
+	}
+
 	return b, err
 }
diff --git a/resources/page/pagemeta/pagemeta_test.go b/resources/page/pagemeta/pagemeta_test.go
new file mode 100644
index 000000000..a66a1f432
--- /dev/null
+++ b/resources/page/pagemeta/pagemeta_test.go
@@ -0,0 +1,64 @@
+// Copyright 2020 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pagemeta
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/gohugoio/hugo/htesting/hqt"
+
+	"github.com/gohugoio/hugo/config"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestDecodeBuildConfig(t *testing.T) {
+	t.Parallel()
+
+	c := qt.New(t)
+
+	configTempl := `
+[_build]
+render = true
+list = %s
+publishResources = true`
+
+	for _, test := range []struct {
+		list   interface{}
+		expect string
+	}{
+		{"true", Always},
+		{"false", Never},
+		{`"always"`, Always},
+		{`"local"`, ListLocally},
+		{`"asdfadf"`, Always},
+	} {
+		cfg, err := config.FromConfigString(fmt.Sprintf(configTempl, test.list), "toml")
+		c.Assert(err, qt.IsNil)
+		bcfg, err := DecodeBuildConfig(cfg.Get("_build"))
+		c.Assert(err, qt.IsNil)
+
+		eq := qt.CmpEquals(hqt.DeepAllowUnexported(BuildConfig{}))
+
+		c.Assert(bcfg, eq, BuildConfig{
+			Render:           true,
+			List:             test.expect,
+			PublishResources: true,
+			set:              true,
+		})
+
+	}
+
+}