Make Page an interface

The main motivation of this commit is to add a `page.Page` interface to replace the very file-oriented `hugolib.Page` struct.
This is all a preparation step for issue  #5074, "pages from other data sources".

But this also fixes a set of annoying limitations, especially related to custom output formats, and shortcodes.

Most notable changes:

* The inner content of shortcodes using the `{{%` as the outer-most delimiter will now be sent to the content renderer, e.g. Blackfriday.
  This means that any markdown will partake in the global ToC and footnote context etc.
* The Custom Output formats are now "fully virtualized". This removes many of the current limitations.
* The taxonomy list type now has a reference to the `Page` object.
  This improves the taxonomy template `.Title` situation and make common template constructs much simpler.

See #5074
Fixes #5763
Fixes #5758
Fixes #5090
Fixes #5204
Fixes #4695
Fixes #5607
Fixes #5707
Fixes #5719
Fixes #3113
Fixes #5706
Fixes #5767
Fixes #5723
Fixes #5769
Fixes #5770
Fixes #5771
Fixes #5759
Fixes #5776
Fixes #5777
Fixes #5778

4 files changed, 177 insertions, 6 deletions

diff --git a/hugofs/createcounting_fs.go b/hugofs/createcounting_fs.go
new file mode 100644
index 000000000..802806b7a
--- /dev/null
+++ b/hugofs/createcounting_fs.go
@@ -0,0 +1,99 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/spf13/afero"
+)
+
+// Reseter is implemented by some of the stateful filesystems.
+type Reseter interface {
+	Reset()
+}
+
+// DuplicatesReporter reports about duplicate filenames.
+type DuplicatesReporter interface {
+	ReportDuplicates() string
+}
+
+func NewCreateCountingFs(fs afero.Fs) afero.Fs {
+	return &createCountingFs{Fs: fs, fileCount: make(map[string]int)}
+}
+
+// ReportDuplicates reports filenames written more than once.
+func (c *createCountingFs) ReportDuplicates() string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	var dupes []string
+
+	for k, v := range c.fileCount {
+		if v > 1 {
+			dupes = append(dupes, fmt.Sprintf("%s (%d)", k, v))
+		}
+	}
+
+	if len(dupes) == 0 {
+		return ""
+	}
+
+	sort.Strings(dupes)
+
+	return strings.Join(dupes, ", ")
+}
+
+// createCountingFs counts filenames of created files or files opened
+// for writing.
+type createCountingFs struct {
+	afero.Fs
+
+	mu        sync.Mutex
+	fileCount map[string]int
+}
+
+func (c *createCountingFs) Reset() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.fileCount = make(map[string]int)
+}
+
+func (fs *createCountingFs) onCreate(filename string) {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+
+	fs.fileCount[filename] = fs.fileCount[filename] + 1
+}
+
+func (fs *createCountingFs) Create(name string) (afero.File, error) {
+	f, err := fs.Fs.Create(name)
+	if err == nil {
+		fs.onCreate(name)
+	}
+	return f, err
+}
+
+func (fs *createCountingFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
+	f, err := fs.Fs.OpenFile(name, flag, perm)
+	if err == nil && isWrite(flag) {
+		fs.onCreate(name)
+	}
+	return f, err
+}
diff --git a/hugofs/fs.go b/hugofs/fs.go
index 52e27bd12..38590a64e 100644
--- a/hugofs/fs.go
+++ b/hugofs/fs.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,6 +15,8 @@
 package hugofs
 
 import (
+	"os"
+
 	"github.com/gohugoio/hugo/config"
 	"github.com/spf13/afero"
 )
@@ -80,3 +82,7 @@ func getWorkingDirFs(base afero.Fs, cfg config.Provider) *afero.BasePathFs {
 
 	return nil
 }
+
+func isWrite(flag int) bool {
+	return flag&os.O_RDWR != 0 || flag&os.O_WRONLY != 0
+}
diff --git a/hugofs/hashing_fs.go b/hugofs/hashing_fs.go
index 2de027ce2..94a50b960 100644
--- a/hugofs/hashing_fs.go
+++ b/hugofs/hashing_fs.go
@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -67,10 +67,6 @@ func (fs *md5HashingFs) wrapFile(f afero.File) afero.File {
 	return &hashingFile{File: f, h: md5.New(), hashReceiver: fs.hashReceiver}
 }
 
-func isWrite(flag int) bool {
-	return flag&os.O_RDWR != 0 || flag&os.O_WRONLY != 0
-}
-
 func (fs *md5HashingFs) Name() string {
 	return "md5HashingFs"
 }
diff --git a/hugofs/stacktracer_fs.go b/hugofs/stacktracer_fs.go
new file mode 100644
index 000000000..d4db164ca
--- /dev/null
+++ b/hugofs/stacktracer_fs.go
@@ -0,0 +1,70 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"runtime"
+
+	"github.com/gohugoio/hugo/common/types"
+
+	"github.com/spf13/afero"
+)
+
+// Make sure we don't accidently use this in the real Hugo.
+var _ types.DevMarker = (*stacktracerFs)(nil)
+
+// NewStacktracerFs wraps the given fs printing stack traces for file creates
+// matching the given regexp pattern.
+func NewStacktracerFs(fs afero.Fs, pattern string) afero.Fs {
+	return &stacktracerFs{Fs: fs, re: regexp.MustCompile(pattern)}
+}
+
+// stacktracerFs can be used in hard-to-debug development situations where
+// you get some input you don't understand where comes from.
+type stacktracerFs struct {
+	afero.Fs
+
+	// Will print stacktrace for every file creates matching this pattern.
+	re *regexp.Regexp
+}
+
+func (fs *stacktracerFs) DevOnly() {
+}
+
+func (fs *stacktracerFs) onCreate(filename string) {
+	if fs.re.MatchString(filename) {
+		trace := make([]byte, 1500)
+		runtime.Stack(trace, true)
+		fmt.Printf("\n===========\n%q:\n%s\n", filename, trace)
+	}
+}
+
+func (fs *stacktracerFs) Create(name string) (afero.File, error) {
+	f, err := fs.Fs.Create(name)
+	if err == nil {
+		fs.onCreate(name)
+	}
+	return f, err
+}
+
+func (fs *stacktracerFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
+	f, err := fs.Fs.OpenFile(name, flag, perm)
+	if err == nil && isWrite(flag) {
+		fs.onCreate(name)
+	}
+	return f, err
+}