diff options
author | Bjørn Erik Pedersen <[email protected]> | 2019-01-02 12:33:26 +0100 |
---|---|---|
committer | Bjørn Erik Pedersen <[email protected]> | 2019-03-23 18:51:22 +0100 |
commit | 597e418cb02883418f2cebb41400e8e61413f651 (patch) | |
tree | 177ad9c540b2583b6dab138c9f0490d28989c7f7 /hugolib | |
parent | 44f5c1c14cb1f42cc5f01739c289e9cfc83602af (diff) | |
download | hugo-597e418cb02883418f2cebb41400e8e61413f651.tar.gz hugo-597e418cb02883418f2cebb41400e8e61413f651.zip |
Make Page an interface
The main motivation of this commit is to add a `page.Page` interface to replace the very file-oriented `hugolib.Page` struct.
This is all a preparation step for issue #5074, "pages from other data sources".
But this also fixes a set of annoying limitations, especially related to custom output formats, and shortcodes.
Most notable changes:
* The inner content of shortcodes using the `{{%` as the outer-most delimiter will now be sent to the content renderer, e.g. Blackfriday.
This means that any markdown will partake in the global ToC and footnote context etc.
* The Custom Output formats are now "fully virtualized". This removes many of the current limitations.
* The taxonomy list type now has a reference to the `Page` object.
This improves the taxonomy template `.Title` situation and make common template constructs much simpler.
See #5074
Fixes #5763
Fixes #5758
Fixes #5090
Fixes #5204
Fixes #4695
Fixes #5607
Fixes #5707
Fixes #5719
Fixes #3113
Fixes #5706
Fixes #5767
Fixes #5723
Fixes #5769
Fixes #5770
Fixes #5771
Fixes #5759
Fixes #5776
Fixes #5777
Fixes #5778
Diffstat (limited to 'hugolib')
100 files changed, 6705 insertions, 11988 deletions
diff --git a/hugolib/alias.go b/hugolib/alias.go index c44f32dbb..599821c0a 100644 --- a/hugolib/alias.go +++ b/hugolib/alias.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,6 +26,7 @@ import ( "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/publisher" + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/helpers" @@ -55,7 +56,12 @@ func newAliasHandler(t tpl.TemplateFinder, l *loggers.Logger, allowRoot bool) al return aliasHandler{t, l, allowRoot} } -func (a aliasHandler) renderAlias(isXHTML bool, permalink string, page *Page) (io.Reader, error) { +type aliasPage struct { + Permalink string + page.Page +} + +func (a aliasHandler) renderAlias(isXHTML bool, permalink string, p page.Page) (io.Reader, error) { t := "alias" if isXHTML { t = "alias-xhtml" @@ -75,12 +81,9 @@ func (a aliasHandler) renderAlias(isXHTML bool, permalink string, page *Page) (i } } - data := struct { - Permalink string - Page *Page - }{ + data := aliasPage{ permalink, - page, + p, } buffer := new(bytes.Buffer) @@ -91,11 +94,11 @@ func (a aliasHandler) renderAlias(isXHTML bool, permalink string, page *Page) (i return buffer, nil } -func (s *Site) writeDestAlias(path, permalink string, outputFormat output.Format, p *Page) (err error) { +func (s *Site) writeDestAlias(path, permalink string, outputFormat output.Format, p page.Page) (err error) { return s.publishDestAlias(false, path, permalink, outputFormat, p) } -func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, outputFormat output.Format, p *Page) (err error) { +func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, outputFormat output.Format, p page.Page) (err error) { handler := newAliasHandler(s.Tmpl, s.Log, allowRoot) isXHTML := strings.HasSuffix(path, ".xhtml") @@ -126,19 +129,19 @@ func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, outputFo func (a aliasHandler) targetPathAlias(src string) (string, error) { originalAlias := src if len(src) <= 0 { - return "", fmt.Errorf("Alias \"\" is an empty string") + return "", fmt.Errorf("alias \"\" is an empty string") } alias := filepath.Clean(src) components := strings.Split(alias, helpers.FilePathSeparator) if !a.allowRoot && alias == helpers.FilePathSeparator { - return "", fmt.Errorf("Alias \"%s\" resolves to website root directory", originalAlias) + return "", fmt.Errorf("alias \"%s\" resolves to website root directory", originalAlias) } // Validate against directory traversal if components[0] == ".." { - return "", fmt.Errorf("Alias \"%s\" traverses outside the website root directory", originalAlias) + return "", fmt.Errorf("alias \"%s\" traverses outside the website root directory", originalAlias) } // Handle Windows file and directory naming restrictions @@ -171,7 +174,7 @@ func (a aliasHandler) targetPathAlias(src string) (string, error) { for _, m := range msgs { a.log.ERROR.Println(m) } - return "", fmt.Errorf("Cannot create \"%s\": Windows filename restriction", originalAlias) + return "", fmt.Errorf("cannot create \"%s\": Windows filename restriction", originalAlias) } for _, m := range msgs { a.log.INFO.Println(m) diff --git a/hugolib/alias_test.go b/hugolib/alias_test.go index da1b80b70..684e35c9a 100644 --- a/hugolib/alias_test.go +++ b/hugolib/alias_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -50,7 +50,7 @@ func TestAlias(t *testing.T) { b.CreateSites().Build(BuildCfg{}) assert.Equal(1, len(b.H.Sites)) - require.Len(t, b.H.Sites[0].RegularPages, 1) + require.Len(t, b.H.Sites[0].RegularPages(), 1) // the real page b.AssertFileContent("public/page/index.html", "For some moments the old man") diff --git a/hugolib/author.go b/hugolib/author.go deleted file mode 100644 index 0f4327097..000000000 --- a/hugolib/author.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -// AuthorList is a list of all authors and their metadata. -type AuthorList map[string]Author - -// Author contains details about the author of a page. -type Author struct { - GivenName string - FamilyName string - DisplayName string - Thumbnail string - Image string - ShortBio string - LongBio string - Email string - Social AuthorSocial -} - -// AuthorSocial is a place to put social details per author. These are the -// standard keys that themes will expect to have available, but can be -// expanded to any others on a per site basis -// - website -// - github -// - facebook -// - twitter -// - googleplus -// - pinterest -// - instagram -// - youtube -// - linkedin -// - skype -type AuthorSocial map[string]string diff --git a/hugolib/collections.go b/hugolib/collections.go index cf75d3732..a794a9866 100644 --- a/hugolib/collections.go +++ b/hugolib/collections.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,19 +14,13 @@ package hugolib import ( - "fmt" - - "github.com/gohugoio/hugo/resources/resource" - "github.com/gohugoio/hugo/common/collections" + "github.com/gohugoio/hugo/resources/page" ) var ( - _ collections.Grouper = (*Page)(nil) - _ collections.Slicer = (*Page)(nil) - _ collections.Slicer = PageGroup{} - _ collections.Slicer = WeightedPage{} - _ resource.ResourcesConverter = Pages{} + _ collections.Grouper = (*pageState)(nil) + _ collections.Slicer = (*pageState)(nil) ) // collections.Slicer implementations below. We keep these bridge implementations @@ -35,50 +29,8 @@ var ( // Slice is not meant to be used externally. It's a bridge function // for the template functions. See collections.Slice. -func (p *Page) Slice(items interface{}) (interface{}, error) { - return toPages(items) -} - -// Slice is not meant to be used externally. It's a bridge function -// for the template functions. See collections.Slice. -func (p PageGroup) Slice(in interface{}) (interface{}, error) { - switch items := in.(type) { - case PageGroup: - return items, nil - case []interface{}: - groups := make(PagesGroup, len(items)) - for i, v := range items { - g, ok := v.(PageGroup) - if !ok { - return nil, fmt.Errorf("type %T is not a PageGroup", v) - } - groups[i] = g - } - return groups, nil - default: - return nil, fmt.Errorf("invalid slice type %T", items) - } -} - -// Slice is not meant to be used externally. It's a bridge function -// for the template functions. See collections.Slice. -func (p WeightedPage) Slice(in interface{}) (interface{}, error) { - switch items := in.(type) { - case WeightedPages: - return items, nil - case []interface{}: - weighted := make(WeightedPages, len(items)) - for i, v := range items { - g, ok := v.(WeightedPage) - if !ok { - return nil, fmt.Errorf("type %T is not a WeightedPage", v) - } - weighted[i] = g - } - return weighted, nil - default: - return nil, fmt.Errorf("invalid slice type %T", items) - } +func (p *pageState) Slice(items interface{}) (interface{}, error) { + return page.ToPages(items) } // collections.Grouper implementations below @@ -86,19 +38,10 @@ func (p WeightedPage) Slice(in interface{}) (interface{}, error) { // Group creates a PageGroup from a key and a Pages object // This method is not meant for external use. It got its non-typed arguments to satisfy // a very generic interface in the tpl package. -func (p *Page) Group(key interface{}, in interface{}) (interface{}, error) { - pages, err := toPages(in) +func (p *pageState) Group(key interface{}, in interface{}) (interface{}, error) { + pages, err := page.ToPages(in) if err != nil { return nil, err } - return PageGroup{Key: key, Pages: pages}, nil -} - -// ToResources wraps resource.ResourcesConverter -func (pages Pages) ToResources() resource.Resources { - r := make(resource.Resources, len(pages)) - for i, p := range pages { - r[i] = p - } - return r + return page.PageGroup{Key: key, Pages: pages}, nil } diff --git a/hugolib/collections_test.go b/hugolib/collections_test.go index 9cf328a05..bc55bdbe8 100644 --- a/hugolib/collections_test.go +++ b/hugolib/collections_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -40,7 +40,7 @@ title: "Page" b.CreateSites().Build(BuildCfg{}) assert.Equal(1, len(b.H.Sites)) - require.Len(t, b.H.Sites[0].RegularPages, 2) + require.Len(t, b.H.Sites[0].RegularPages(), 2) b.AssertFileContent("public/index.html", "cool: 2") } @@ -79,12 +79,12 @@ tags_weight: %d b.CreateSites().Build(BuildCfg{}) assert.Equal(1, len(b.H.Sites)) - require.Len(t, b.H.Sites[0].RegularPages, 2) + require.Len(t, b.H.Sites[0].RegularPages(), 2) b.AssertFileContent("public/index.html", - "pages:2:hugolib.Pages:Page(/page1.md)/Page(/page2.md)", - "pageGroups:2:hugolib.PagesGroup:Page(/page1.md)/Page(/page2.md)", - `weightedPages:2::hugolib.WeightedPages:[WeightedPage(10,"Page") WeightedPage(20,"Page")]`) + "pages:2:page.Pages:Page(/page1.md)/Page(/page2.md)", + "pageGroups:2:page.PagesGroup:Page(/page1.md)/Page(/page2.md)", + `weightedPages:2::page.WeightedPages:[WeightedPage(10,"Page") WeightedPage(20,"Page")]`) } func TestAppendFunc(t *testing.T) { @@ -129,11 +129,11 @@ tags_weight: %d b.CreateSites().Build(BuildCfg{}) assert.Equal(1, len(b.H.Sites)) - require.Len(t, b.H.Sites[0].RegularPages, 2) + require.Len(t, b.H.Sites[0].RegularPages(), 2) b.AssertFileContent("public/index.html", - "pages:2:hugolib.Pages:Page(/page2.md)/Page(/page1.md)", - "appendPages:9:hugolib.Pages:home/page", + "pages:2:page.Pages:Page(/page2.md)/Page(/page1.md)", + "appendPages:9:page.Pages:home/page", "appendStrings:[]string:[a b c d e]", "appendStringsSlice:[]string:[a b c c d]", "union:[]string:[a b c d e]", diff --git a/hugolib/config.go b/hugolib/config.go index 6a1de32be..50e4ca6ec 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -1,4 +1,4 @@ -// Copyright 2016-present The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ import ( "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib/paths" "github.com/pkg/errors" _errors "github.com/pkg/errors" @@ -177,14 +176,6 @@ type configLoader struct { ConfigSourceDescriptor } -func (l configLoader) wrapFileInfoError(err error, fi os.FileInfo) error { - rfi, ok := fi.(hugofs.RealFilenameInfo) - if !ok { - return err - } - return l.wrapFileError(err, rfi.RealFilename()) -} - func (l configLoader) loadConfig(configName string, v *viper.Viper) (string, error) { baseDir := l.configFileDir() var baseFilename string @@ -240,11 +231,6 @@ func (l configLoader) wrapFileError(err error, filename string) error { return err } -func (l configLoader) newRealBaseFs(path string) afero.Fs { - return hugofs.NewBasePathRealFilenameFs(afero.NewBasePathFs(l.Fs, path).(*afero.BasePathFs)) - -} - func (l configLoader) loadConfigFromConfigDir(v *viper.Viper) ([]string, error) { sourceFs := l.Fs configDir := l.AbsConfigDir @@ -274,7 +260,7 @@ func (l configLoader) loadConfigFromConfigDir(v *viper.Viper) ([]string, error) for _, configDir := range configDirs { err := afero.Walk(sourceFs, configDir, func(path string, fi os.FileInfo, err error) error { - if fi == nil { + if fi == nil || err != nil { return nil } @@ -616,8 +602,8 @@ func loadDefaultSettingsFor(v *viper.Viper) error { v.SetDefault("removePathAccents", false) v.SetDefault("titleCaseStyle", "AP") v.SetDefault("taxonomies", map[string]string{"tag": "tags", "category": "categories"}) - v.SetDefault("permalinks", make(PermalinkOverrides, 0)) - v.SetDefault("sitemap", Sitemap{Priority: -1, Filename: "sitemap.xml"}) + v.SetDefault("permalinks", make(map[string]string)) + v.SetDefault("sitemap", config.Sitemap{Priority: -1, Filename: "sitemap.xml"}) v.SetDefault("pygmentsStyle", "monokai") v.SetDefault("pygmentsUseClasses", false) v.SetDefault("pygmentsCodeFences", false) @@ -625,7 +611,6 @@ func loadDefaultSettingsFor(v *viper.Viper) error { v.SetDefault("pygmentsOptions", "") v.SetDefault("disableLiveReload", false) v.SetDefault("pluralizeListTitles", true) - v.SetDefault("preserveTaxonomyNames", false) v.SetDefault("forceSyncStatic", false) v.SetDefault("footnoteAnchorPrefix", "") v.SetDefault("footnoteReturnLinkContents", "") diff --git a/hugolib/datafiles_test.go b/hugolib/datafiles_test.go index 6685de4cc..b65183a8a 100644 --- a/hugolib/datafiles_test.go +++ b/hugolib/datafiles_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -349,7 +349,7 @@ func doTestDataDirImpl(t *testing.T, dd dataDir, expected interface{}, configKey s := buildSingleSiteExpected(t, false, expectBuildError, depsCfg, BuildCfg{SkipRender: true}) - if !expectBuildError && !reflect.DeepEqual(expected, s.Data) { + if !expectBuildError && !reflect.DeepEqual(expected, s.h.Data()) { // This disabled code detects the situation described in the WARNING message below. // The situation seems to only occur for TOML data with integer values. // Perhaps the TOML parser returns ints in another type. @@ -366,7 +366,7 @@ func doTestDataDirImpl(t *testing.T, dd dataDir, expected interface{}, configKey } */ - return fmt.Sprintf("Expected data:\n%v got\n%v\n\nExpected type structure:\n%#[1]v got\n%#[2]v", expected, s.Data) + return fmt.Sprintf("Expected data:\n%v got\n%v\n\nExpected type structure:\n%#[1]v got\n%#[2]v", expected, s.h.Data()) } return diff --git a/hugolib/disableKinds_test.go b/hugolib/disableKinds_test.go index edada1419..f5c093646 100644 --- a/hugolib/disableKinds_test.go +++ b/hugolib/disableKinds_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ import ( "fmt" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/deps" "github.com/spf13/afero" @@ -33,13 +35,13 @@ func TestDisableKindsNoneDisabled(t *testing.T) { func TestDisableKindsSomeDisabled(t *testing.T) { t.Parallel() - doTestDisableKinds(t, KindSection, kind404) + doTestDisableKinds(t, page.KindSection, kind404) } func TestDisableKindsOneDisabled(t *testing.T) { t.Parallel() for _, kind := range allKinds { - if kind == KindPage { + if kind == page.KindPage { // Turning off regular page generation have some side-effects // not handled by the assertions below (no sections), so // skip that for now. @@ -124,64 +126,64 @@ func assertDisabledKinds(th testHelper, s *Site, disabled ...string) { assertDisabledKind(th, func(isDisabled bool) bool { if isDisabled { - return len(s.RegularPages) == 0 + return len(s.RegularPages()) == 0 } - return len(s.RegularPages) > 0 - }, disabled, KindPage, "public/sect/p1/index.html", "Single|P1") + return len(s.RegularPages()) > 0 + }, disabled, page.KindPage, "public/sect/p1/index.html", "Single|P1") assertDisabledKind(th, func(isDisabled bool) bool { - p := s.getPage(KindHome) + p := s.getPage(page.KindHome) if isDisabled { return p == nil } return p != nil - }, disabled, KindHome, "public/index.html", "Home") + }, disabled, page.KindHome, "public/index.html", "Home") assertDisabledKind(th, func(isDisabled bool) bool { - p := s.getPage(KindSection, "sect") + p := s.getPage(page.KindSection, "sect") if isDisabled { return p == nil } return p != nil - }, disabled, KindSection, "public/sect/index.html", "Sects") + }, disabled, page.KindSection, "public/sect/index.html", "Sects") assertDisabledKind(th, func(isDisabled bool) bool { - p := s.getPage(KindTaxonomy, "tags", "tag1") + p := s.getPage(page.KindTaxonomy, "tags", "tag1") if isDisabled { return p == nil } return p != nil - }, disabled, KindTaxonomy, "public/tags/tag1/index.html", "Tag1") + }, disabled, page.KindTaxonomy, "public/tags/tag1/index.html", "Tag1") assertDisabledKind(th, func(isDisabled bool) bool { - p := s.getPage(KindTaxonomyTerm, "tags") + p := s.getPage(page.KindTaxonomyTerm, "tags") if isDisabled { return p == nil } return p != nil - }, disabled, KindTaxonomyTerm, "public/tags/index.html", "Tags") + }, disabled, page.KindTaxonomyTerm, "public/tags/index.html", "Tags") assertDisabledKind(th, func(isDisabled bool) bool { - p := s.getPage(KindTaxonomyTerm, "categories") + p := s.getPage(page.KindTaxonomyTerm, "categories") if isDisabled { return p == nil } return p != nil - }, disabled, KindTaxonomyTerm, "public/categories/index.html", "Category Terms") + }, disabled, page.KindTaxonomyTerm, "public/categories/index.html", "Category Terms") assertDisabledKind(th, func(isDisabled bool) bool { - p := s.getPage(KindTaxonomy, "categories", "hugo") + p := s.getPage(page.KindTaxonomy, "categories", "hugo") if isDisabled { return p == nil } return p != nil - }, disabled, KindTaxonomy, "public/categories/hugo/index.html", "Hugo") + }, disabled, page.KindTaxonomy, "public/categories/hugo/index.html", "Hugo") // The below have no page in any collection. assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kindRSS, "public/index.xml", "<link>") assertDisabledKind(th, func(isDisabled bool) bool { return true }, disabled, kindSitemap, "public/sitemap.xml", "sitemap") @@ -195,7 +197,7 @@ func assertDisabledKind(th testHelper, kindAssert func(bool) bool, disabled []st if kind == kindRSS && !isDisabled { // If the home page is also disabled, there is not RSS to look for. - if stringSliceContains(KindHome, disabled...) { + if stringSliceContains(page.KindHome, disabled...) { isDisabled = true } } diff --git a/hugolib/embedded_shortcodes_test.go b/hugolib/embedded_shortcodes_test.go index 3a6220b53..c70380a4b 100644 --- a/hugolib/embedded_shortcodes_test.go +++ b/hugolib/embedded_shortcodes_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,8 @@ import ( "strings" "testing" + "github.com/spf13/cast" + "path/filepath" "github.com/gohugoio/hugo/deps" @@ -67,9 +69,11 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) { s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - require.Len(t, s.RegularPages, 1) + require.Len(t, s.RegularPages(), 1) - output := string(s.RegularPages[0].content()) + content, err := s.RegularPages()[0].Content() + require.NoError(t, err) + output := cast.ToString(content) if !strings.Contains(output, expected) { t.Errorf("Got\n%q\nExpected\n%q", output, expected) diff --git a/hugolib/gitinfo.go b/hugolib/gitinfo.go index d356fcf07..6acc47d17 100644 --- a/hugolib/gitinfo.go +++ b/hugolib/gitinfo.go @@ -1,4 +1,4 @@ -// Copyright 2016-present The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import ( "github.com/bep/gitmap" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/resources/page" ) type gitInfo struct { @@ -26,15 +27,12 @@ type gitInfo struct { repo *gitmap.GitRepo } -func (g *gitInfo) forPage(p *Page) (*gitmap.GitInfo, bool) { - if g == nil { - return nil, false - } - - name := strings.TrimPrefix(filepath.ToSlash(p.Filename()), g.contentDir) +func (g *gitInfo) forPage(p page.Page) *gitmap.GitInfo { + name := strings.TrimPrefix(filepath.ToSlash(p.File().Filename()), g.contentDir) name = strings.TrimPrefix(name, "/") - return g.repo.Files[name], true + return g.repo.Files[name] + } func newGitInfo(cfg config.Provider) (*gitInfo, error) { diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 9ce1c438e..af1e0fbac 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,14 +14,24 @@ package hugolib import ( - "errors" "io" "path/filepath" "sort" "strings" "sync" + "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/parser/metadecoders" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/source" + + "github.com/bep/gitmap" "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" "github.com/gohugoio/hugo/publisher" @@ -30,8 +40,10 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/lazy" "github.com/gohugoio/hugo/i18n" + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/tpl/tplimpl" ) @@ -48,17 +60,96 @@ type HugoSites struct { // If this is running in the dev server. running bool + // Render output formats for all sites. + renderFormats output.Formats + *deps.Deps + gitInfo *gitInfo + + // As loaded from the /data dirs + data map[string]interface{} + // Keeps track of bundle directories and symlinks to enable partial rebuilding. ContentChanges *contentChangeMap - // If enabled, keeps a revision map for all content. - gitInfo *gitInfo + init *hugoSitesInit + + *fatalErrorHandler +} + +type fatalErrorHandler struct { + mu sync.Mutex + + h *HugoSites + + err error + + done bool + donec chan bool // will be closed when done +} + +// FatalError error is used in some rare situations where it does not make sense to +// continue processing, to abort as soon as possible and log the error. +func (f *fatalErrorHandler) FatalError(err error) { + f.mu.Lock() + defer f.mu.Unlock() + if !f.done { + f.done = true + close(f.donec) + } + f.err = err } -func (h *HugoSites) siteInfos() SiteInfos { - infos := make(SiteInfos, len(h.Sites)) +func (f *fatalErrorHandler) getErr() error { + f.mu.Lock() + defer f.mu.Unlock() + return f.err +} + +func (f *fatalErrorHandler) Done() <-chan bool { + return f.donec +} + +type hugoSitesInit struct { + // Loads the data from all of the /data folders. + data *lazy.Init + + // Loads the Git info for all the pages if enabled. + gitInfo *lazy.Init + + // Maps page translations. + translations *lazy.Init +} + +func (h *hugoSitesInit) Reset() { + h.data.Reset() + h.gitInfo.Reset() + h.translations.Reset() +} + +func (h *HugoSites) Data() map[string]interface{} { + if _, err := h.init.data.Do(); err != nil { + h.SendError(errors.Wrap(err, "failed to load data")) + return nil + } + return h.data +} + +func (h *HugoSites) gitInfoForPage(p page.Page) (*gitmap.GitInfo, error) { + if _, err := h.init.gitInfo.Do(); err != nil { + return nil, err + } + + if h.gitInfo == nil { + return nil, nil + } + + return h.gitInfo.forPage(p), nil +} + +func (h *HugoSites) siteInfos() page.Sites { + infos := make(page.Sites, len(h.Sites)) for i, site := range h.Sites { infos[i] = &site.Info } @@ -106,7 +197,7 @@ func (h *HugoSites) IsMultihost() bool { func (h *HugoSites) LanguageSet() map[string]bool { set := make(map[string]bool) for _, s := range h.Sites { - set[s.Language.Lang] = true + set[s.language.Lang] = true } return set } @@ -129,14 +220,14 @@ func (h *HugoSites) PrintProcessingStats(w io.Writer) { func (h *HugoSites) langSite() map[string]*Site { m := make(map[string]*Site) for _, s := range h.Sites { - m[s.Language.Lang] = s + m[s.language.Lang] = s } return m } // GetContentPage finds a Page with content given the absolute filename. // Returns nil if none found. -func (h *HugoSites) GetContentPage(filename string) *Page { +func (h *HugoSites) GetContentPage(filename string) page.Page { for _, s := range h.Sites { pos := s.rawAllPages.findPagePosByFilename(filename) if pos == -1 { @@ -178,10 +269,40 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { running: cfg.Running, multilingual: langConfig, multihost: cfg.Cfg.GetBool("multihost"), - Sites: sites} + Sites: sites, + init: &hugoSitesInit{ + data: lazy.New(), + gitInfo: lazy.New(), + translations: lazy.New(), + }, + } + + h.fatalErrorHandler = &fatalErrorHandler{ + h: h, + donec: make(chan bool), + } + + h.init.data.Add(func() (interface{}, error) { + err := h.loadData(h.PathSpec.BaseFs.Data.Fs) + return err, nil + }) + + h.init.translations.Add(func() (interface{}, error) { + if len(h.Sites) > 1 { + allTranslations := pagesToTranslationsMap(h.Sites) + assignTranslationsToPages(allTranslations, h.Sites) + } + + return nil, nil + }) + + h.init.gitInfo.Add(func() (interface{}, error) { + err := h.loadGitInfo() + return nil, err + }) for _, s := range sites { - s.owner = h + s.h = h } if err := applyDeps(cfg, sites...); err != nil { @@ -197,14 +318,10 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { h.ContentChanges = contentChangeTracker } - if err := h.initGitInfo(); err != nil { - return nil, err - } - return h, nil } -func (h *HugoSites) initGitInfo() error { +func (h *HugoSites) loadGitInfo() error { if h.Cfg.GetBool("enableGitInfo") { gi, err := newGitInfo(h.Cfg) if err != nil { @@ -247,16 +364,16 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { d.Site = &s.Info - siteConfig, err := loadSiteConfig(s.Language) + siteConfig, err := loadSiteConfig(s.language) if err != nil { return err } - s.siteConfig = siteConfig - s.siteRefLinker, err = newSiteRefLinker(s.Language, s) + s.siteConfigConfig = siteConfig + s.siteRefLinker, err = newSiteRefLinker(s.language, s) return err } - cfg.Language = s.Language + cfg.Language = s.language cfg.MediaTypes = s.mediaTypesConfig cfg.OutputFormats = s.outputFormatsConfig @@ -347,11 +464,23 @@ func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) { return sites, nil } -// Reset resets the sites and template caches, making it ready for a full rebuild. -func (h *HugoSites) reset() { - for i, s := range h.Sites { - h.Sites[i] = s.reset() +// Reset resets the sites and template caches etc., making it ready for a full rebuild. +func (h *HugoSites) reset(config *BuildCfg) { + if config.ResetState { + for i, s := range h.Sites { + h.Sites[i] = s.reset() + if r, ok := s.Fs.Destination.(hugofs.Reseter); ok { + r.Reset() + } + } } + + h.fatalErrorHandler = &fatalErrorHandler{ + h: h, + donec: make(chan bool), + } + + h.init.Reset() } // resetLogs resets the log counters etc. Used to do a new build on the same sites. @@ -387,7 +516,7 @@ func (h *HugoSites) createSitesFromConfig(cfg config.Provider) error { h.Sites = sites for _, s := range sites { - s.owner = h + s.h = h } if err := applyDeps(depsCfg, sites...); err != nil { @@ -435,7 +564,10 @@ type BuildCfg struct { // Note that a page does not have to have a content page / file. // For regular builds, this will allways return true. // TODO(bep) rename/work this. -func (cfg *BuildCfg) shouldRender(p *Page) bool { +func (cfg *BuildCfg) shouldRender(p *pageState) bool { + if !p.render { + return false + } if p.forceRender { p.forceRender = false return true @@ -445,15 +577,8 @@ func (cfg *BuildCfg) shouldRender(p *Page) bool { return true } - if cfg.RecentlyVisited[p.RelPermalink()] { - if cfg.PartialReRender { - _ = p.initMainOutputFormat() - } - return true - } - - if cfg.whatChanged != nil && p.File != nil { - return cfg.whatChanged.files[p.File.Filename()] + if cfg.whatChanged != nil && p.File() != nil { + return cfg.whatChanged.files[p.File().Filename()] } return false @@ -477,100 +602,85 @@ func (h *HugoSites) renderCrossSitesArtifacts() error { return nil } - // TODO(bep) DRY - sitemapDefault := parseSitemap(h.Cfg.GetStringMap("sitemap")) - s := h.Sites[0] smLayouts := []string{"sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml"} return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemapindex", - sitemapDefault.Filename, h.toSiteInfos(), smLayouts...) -} - -func (h *HugoSites) assignMissingTranslations() error { - - // This looks heavy, but it should be a small number of nodes by now. - allPages := h.findAllPagesByKindNotIn(KindPage) - for _, nodeType := range []string{KindHome, KindSection, KindTaxonomy, KindTaxonomyTerm} { - nodes := h.findPagesByKindIn(nodeType, allPages) - - // Assign translations - for _, t1 := range nodes { - for _, t2 := range nodes { - if t1.isNewTranslation(t2) { - t1.translations = append(t1.translations, t2) - } - } - } - } - - // Now we can sort the translations. - for _, p := range allPages { - if len(p.translations) > 0 { - pageBy(languagePageSort).Sort(p.translations) - } - } - return nil - + s.siteCfg.sitemap.Filename, h.toSiteInfos(), smLayouts...) } // createMissingPages creates home page, taxonomies etc. that isnt't created as an // effect of having a content file. func (h *HugoSites) createMissingPages() error { - var newPages Pages + var newPages pageStatePages for _, s := range h.Sites { - if s.isEnabled(KindHome) { + if s.isEnabled(page.KindHome) { // home pages - home := s.findPagesByKind(KindHome) - if len(home) > 1 { + homes := s.findWorkPagesByKind(page.KindHome) + if len(homes) > 1 { panic("Too many homes") } - if len(home) == 0 { - n := s.newHomePage() - s.Pages = append(s.Pages, n) - newPages = append(newPages, n) + var home *pageState + if len(homes) == 0 { + home = s.newPage(page.KindHome) + s.workAllPages = append(s.workAllPages, home) + newPages = append(newPages, home) + } else { + home = homes[0] } + + s.home = home } // Will create content-less root sections. newSections := s.assembleSections() - s.Pages = append(s.Pages, newSections...) + s.workAllPages = append(s.workAllPages, newSections...) newPages = append(newPages, newSections...) + taxonomyTermEnabled := s.isEnabled(page.KindTaxonomyTerm) + taxonomyEnabled := s.isEnabled(page.KindTaxonomy) + // taxonomy list and terms pages - taxonomies := s.Language.GetStringMapString("taxonomies") + taxonomies := s.Language().GetStringMapString("taxonomies") if len(taxonomies) > 0 { - taxonomyPages := s.findPagesByKind(KindTaxonomy) - taxonomyTermsPages := s.findPagesByKind(KindTaxonomyTerm) + taxonomyPages := s.findWorkPagesByKind(page.KindTaxonomy) + taxonomyTermsPages := s.findWorkPagesByKind(page.KindTaxonomyTerm) + + // Make them navigable from WeightedPage etc. + for _, p := range taxonomyPages { + p.getTaxonomyNodeInfo().TransferValues(p) + } + for _, p := range taxonomyTermsPages { + p.getTaxonomyNodeInfo().TransferValues(p) + } + for _, plural := range taxonomies { - if s.isEnabled(KindTaxonomyTerm) { + if taxonomyTermEnabled { foundTaxonomyTermsPage := false for _, p := range taxonomyTermsPages { - if p.sectionsPath() == plural { + if p.SectionsPath() == plural { foundTaxonomyTermsPage = true break } } if !foundTaxonomyTermsPage { - n := s.newTaxonomyTermsPage(plural) - s.Pages = append(s.Pages, n) + n := s.newPage(page.KindTaxonomyTerm, plural) + n.getTaxonomyNodeInfo().TransferValues(n) + s.workAllPages = append(s.workAllPages, n) newPages = append(newPages, n) } } - if s.isEnabled(KindTaxonomy) { - for key := range s.Taxonomies[plural] { + if taxonomyEnabled { + for termKey := range s.Taxonomies[plural] { + foundTaxonomyPage := false - origKey := key - if s.Info.preserveTaxonomyNames { - key = s.PathSpec.MakePathSanitized(key) - } for _, p := range taxonomyPages { - sectionsPath := p.sectionsPath() + sectionsPath := p.SectionsPath() if !strings.HasPrefix(sectionsPath, plural) { continue @@ -579,20 +689,21 @@ func (h *HugoSites) createMissingPages() error { singularKey := strings.TrimPrefix(sectionsPath, plural) singularKey = strings.TrimPrefix(singularKey, "/") - // Some people may have /authors/MaxMustermann etc. as paths. - // p.sections contains the raw values from the file system. - // See https://github.com/gohugoio/hugo/issues/4238 - singularKey = s.PathSpec.MakePathSanitized(singularKey) - - if singularKey == key { + if singularKey == termKey { foundTaxonomyPage = true break } } if !foundTaxonomyPage { - n := s.newTaxonomyPage(plural, origKey) - s.Pages = append(s.Pages, n) + info := s.taxonomyNodes.Get(plural, termKey) + if info == nil { + panic("no info found") + } + + n := s.newTaxonomyPage(info.term, info.plural, info.termKey) + info.TransferValues(n) + s.workAllPages = append(s.workAllPages, n) newPages = append(newPages, n) } } @@ -601,24 +712,6 @@ func (h *HugoSites) createMissingPages() error { } } - if len(newPages) > 0 { - // This resorting is unfortunate, but it also needs to be sorted - // when sections are created. - first := h.Sites[0] - - first.AllPages = append(first.AllPages, newPages...) - - first.AllPages.sort() - - for _, s := range h.Sites { - s.Pages.sort() - } - - for i := 1; i < len(h.Sites); i++ { - h.Sites[i].AllPages = first.AllPages - } - } - return nil } @@ -628,61 +721,58 @@ func (h *HugoSites) removePageByFilename(filename string) { } } -func (h *HugoSites) setupTranslations() { +func (h *HugoSites) createPageCollections() error { for _, s := range h.Sites { for _, p := range s.rawAllPages { - if p.Kind == kindUnknown { - p.Kind = p.kindFromSections() - } - - if !p.s.isEnabled(p.Kind) { + if !s.isEnabled(p.Kind()) { continue } - shouldBuild := p.shouldBuild() - s.updateBuildStats(p) + shouldBuild := s.shouldBuild(p) + s.buildStats.update(p) if shouldBuild { - if p.headless { + if p.m.headless { s.headlessPages = append(s.headlessPages, p) } else { - s.Pages = append(s.Pages, p) + s.workAllPages = append(s.workAllPages, p) } } } } - allPages := make(Pages, 0) + allPages := newLazyPagesFactory(func() page.Pages { + var pages page.Pages + for _, s := range h.Sites { + pages = append(pages, s.Pages()...) + } - for _, s := range h.Sites { - allPages = append(allPages, s.Pages...) - } + page.SortByDefault(pages) - allPages.sort() + return pages + }) - for _, s := range h.Sites { - s.AllPages = allPages - } + allRegularPages := newLazyPagesFactory(func() page.Pages { + return h.findPagesByKindIn(page.KindPage, allPages.get()) + }) - // Pull over the collections from the master site - for i := 1; i < len(h.Sites); i++ { - h.Sites[i].Data = h.Sites[0].Data + for _, s := range h.Sites { + s.PageCollections.allPages = allPages + s.PageCollections.allRegularPages = allRegularPages } - if len(h.Sites) > 1 { - allTranslations := pagesToTranslationsMap(allPages) - assignTranslationsToPages(allTranslations, allPages) - } + return nil } -func (s *Site) preparePagesForRender(start bool) error { - for _, p := range s.Pages { - if err := p.prepareForRender(start); err != nil { +func (s *Site) preparePagesForRender(idx int) error { + + for _, p := range s.workAllPages { + if err := p.initOutputFormat(idx); err != nil { return err } } for _, p := range s.headlessPages { - if err := p.prepareForRender(start); err != nil { + if err := p.initOutputFormat(idx); err != nil { return err } } @@ -691,62 +781,141 @@ func (s *Site) preparePagesForRender(start bool) error { } // Pages returns all pages for all sites. -func (h *HugoSites) Pages() Pages { - return h.Sites[0].AllPages +func (h *HugoSites) Pages() page.Pages { + return h.Sites[0].AllPages() } -func handleShortcodes(p *PageWithoutContent, rawContentCopy []byte) ([]byte, error) { - if p.shortcodeState != nil && p.shortcodeState.contentShortcodes.Len() > 0 { - p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", p.shortcodeState.contentShortcodes.Len(), p.BaseFileName()) - err := p.shortcodeState.executeShortcodesForDelta(p) +func (h *HugoSites) loadData(fs afero.Fs) (err error) { + spec := source.NewSourceSpec(h.PathSpec, fs) + fileSystem := spec.NewFilesystem("") + h.data = make(map[string]interface{}) + for _, r := range fileSystem.Files() { + if err := h.handleDataFile(r); err != nil { + return err + } + } - if err != nil { + return +} + +func (h *HugoSites) handleDataFile(r source.ReadableFile) error { + var current map[string]interface{} - return rawContentCopy, err + f, err := r.Open() + if err != nil { + return errors.Wrapf(err, "Failed to open data file %q:", r.LogicalName()) + } + defer f.Close() + + // Crawl in data tree to insert data + current = h.data + keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator) + // The first path element is the virtual folder (typically theme name), which is + // not part of the key. + if len(keyParts) > 1 { + for _, key := range keyParts[1:] { + if key != "" { + if _, ok := current[key]; !ok { + current[key] = make(map[string]interface{}) + } + current = current[key].(map[string]interface{}) + } } + } - rawContentCopy, err = replaceShortcodeTokens(rawContentCopy, shortcodePlaceholderPrefix, p.shortcodeState.renderedShortcodes) + data, err := h.readData(r) + if err != nil { + return h.errWithFileContext(err, r) + } - if err != nil { - p.s.Log.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error()) + if data == nil { + return nil + } + + // filepath.Walk walks the files in lexical order, '/' comes before '.' + // this warning could happen if + // 1. A theme uses the same key; the main data folder wins + // 2. A sub folder uses the same key: the sub folder wins + higherPrecedentData := current[r.BaseFileName()] + + switch data.(type) { + case nil: + // hear the crickets? + + case map[string]interface{}: + + switch higherPrecedentData.(type) { + case nil: + current[r.BaseFileName()] = data + case map[string]interface{}: + // merge maps: insert entries from data for keys that + // don't already exist in higherPrecedentData + higherPrecedentMap := higherPrecedentData.(map[string]interface{}) + for key, value := range data.(map[string]interface{}) { + if _, exists := higherPrecedentMap[key]; exists { + h.Log.WARN.Printf("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path()) + } else { + higherPrecedentMap[key] = value + } + } + default: + // can't merge: higherPrecedentData is not a map + h.Log.WARN.Printf("The %T data from '%s' overridden by "+ + "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData) + } + + case []interface{}: + if higherPrecedentData == nil { + current[r.BaseFileName()] = data + } else { + // we don't merge array data + h.Log.WARN.Printf("The %T data from '%s' overridden by "+ + "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData) } + + default: + h.Log.ERROR.Printf("unexpected data type %T in file %s", data, r.LogicalName()) } - return rawContentCopy, nil + return nil } -func (s *Site) updateBuildStats(page *Page) { - if page.IsDraft() { - s.draftCount++ +func (h *HugoSites) errWithFileContext(err error, f source.File) error { + rfi, ok := f.FileInfo().(hugofs.RealFilenameInfo) + if !ok { + return err } - if page.IsFuture() { - s.futureCount++ - } + realFilename := rfi.RealFilename() - if page.IsExpired() { - s.expiredCount++ - } -} + err, _ = herrors.WithFileContextForFile( + err, + realFilename, + realFilename, + h.SourceSpec.Fs.Source, + herrors.SimpleLineMatcher) -func (h *HugoSites) findPagesByKindNotIn(kind string, inPages Pages) Pages { - return h.Sites[0].findPagesByKindNotIn(kind, inPages) + return err } -func (h *HugoSites) findPagesByKindIn(kind string, inPages Pages) Pages { - return h.Sites[0].findPagesByKindIn(kind, inPages) -} +func (h *HugoSites) readData(f source.ReadableFile) (interface{}, error) { + file, err := f.Open() + if err != nil { + return nil, errors.Wrap(err, "readData: failed to open data file") + } + defer file.Close() + content := helpers.ReaderToBytes(file) -func (h *HugoSites) findAllPagesByKind(kind string) Pages { - return h.findPagesByKindIn(kind, h.Sites[0].AllPages) + format := metadecoders.FormatFromString(f.Extension()) + return metadecoders.Default.Unmarshal(content, format) } -func (h *HugoSites) findAllPagesByKindNotIn(kind string) Pages { - return h.findPagesByKindNotIn(kind, h.Sites[0].AllPages) +func (h *HugoSites) findPagesByKindIn(kind string, inPages page.Pages) page.Pages { + return h.Sites[0].findPagesByKindIn(kind, inPages) } -func (h *HugoSites) findPagesByShortcode(shortcode string) Pages { - var pages Pages +func (h *HugoSites) findPagesByShortcode(shortcode string) page.Pages { + var pages page.Pages for _, s := range h.Sites { pages = append(pages, s.findPagesByShortcode(shortcode)...) } diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index ec5070fa8..214f72c5f 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -1,4 +1,4 @@ -// Copyright 2016-present The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,12 @@ package hugolib import ( "bytes" + "context" "fmt" + "runtime/trace" + "sort" + + "github.com/gohugoio/hugo/output" "errors" @@ -26,6 +31,9 @@ import ( // Build builds all sites. If filesystem events are provided, // this is considered to be a potential partial rebuild. func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { + ctx, task := trace.NewTask(context.Background(), "Build") + defer task.End() + errCollector := h.StartErrorCollector() errs := make(chan error) @@ -71,22 +79,36 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { return err } } else { - if err := h.init(conf); err != nil { + if err := h.initSites(conf); err != nil { return err } } - if err := h.process(conf, events...); err != nil { + var err error + + f := func() { + err = h.process(conf, events...) + } + trace.WithRegion(ctx, "process", f) + if err != nil { return err } - if err := h.assemble(conf); err != nil { + f = func() { + err = h.assemble(conf) + } + trace.WithRegion(ctx, "assemble", f) + if err != nil { return err } + return nil } - prepareErr = prepare() + f := func() { + prepareErr = prepare() + } + trace.WithRegion(ctx, "prepare", f) if prepareErr != nil { h.SendError(prepareErr) } @@ -94,7 +116,12 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { } if prepareErr == nil { - if err := h.render(conf); err != nil { + var err error + f := func() { + err = h.render(conf) + } + trace.WithRegion(ctx, "render", f) + if err != nil { h.SendError(err) } } @@ -120,6 +147,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { return err } + if err := h.fatalErrorHandler.getErr(); err != nil { + return err + } + errorCount := h.Log.ErrorCounter.Count() if errorCount > 0 { return fmt.Errorf("logged %d error(s)", errorCount) @@ -132,17 +163,8 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { // Build lifecycle methods below. // The order listed matches the order of execution. -func (h *HugoSites) init(config *BuildCfg) error { - - for _, s := range h.Sites { - if s.PageCollections == nil { - s.PageCollections = newPageCollections() - } - } - - if config.ResetState { - h.reset() - } +func (h *HugoSites) initSites(config *BuildCfg) error { + h.reset(config) if config.NewConfig != nil { if err := h.createSitesFromConfig(config.NewConfig); err != nil { @@ -155,28 +177,22 @@ func (h *HugoSites) init(config *BuildCfg) error { func (h *HugoSites) initRebuild(config *BuildCfg) error { if config.NewConfig != nil { - return errors.New("Rebuild does not support 'NewConfig'.") + return errors.New("rebuild does not support 'NewConfig'") } if config.ResetState { - return errors.New("Rebuild does not support 'ResetState'.") + return errors.New("rebuild does not support 'ResetState'") } if !h.running { - return errors.New("Rebuild called when not in watch mode") - } - - if config.whatChanged.source { - // This is for the non-renderable content pages (rarely used, I guess). - // We could maybe detect if this is really needed, but it should be - // pretty fast. - h.TemplateHandler().RebuildClone() + return errors.New("rebuild called when not in watch mode") } for _, s := range h.Sites { s.resetBuildState() } + h.reset(config) h.resetLogs() helpers.InitLoggers() @@ -203,14 +219,6 @@ func (h *HugoSites) process(config *BuildCfg, events ...fsnotify.Event) error { } func (h *HugoSites) assemble(config *BuildCfg) error { - if config.whatChanged.source { - for _, s := range h.Sites { - s.createTaxonomiesEntries() - } - } - - // TODO(bep) we could probably wait and do this in one go later - h.setupTranslations() if len(h.Sites) > 1 { // The first is initialized during process; initialize the rest @@ -221,47 +229,26 @@ func (h *HugoSites) assemble(config *BuildCfg) error { } } + if err := h.createPageCollections(); err != nil { + return err + } + if config.whatChanged.source { for _, s := range h.Sites { - if err := s.buildSiteMeta(); err != nil { + if err := s.assembleTaxonomies(); err != nil { return err } } } + // Create pagexs for the section pages etc. without content file. if err := h.createMissingPages(); err != nil { return err } for _, s := range h.Sites { - for _, pages := range []Pages{s.Pages, s.headlessPages} { - for _, p := range pages { - // May have been set in front matter - if len(p.outputFormats) == 0 { - p.outputFormats = s.outputFormats[p.Kind] - } - - if p.headless { - // headless = 1 output format only - p.outputFormats = p.outputFormats[:1] - } - for _, r := range p.Resources.ByType(pageResourceType) { - r.(*Page).outputFormats = p.outputFormats - } - - if err := p.initPaths(); err != nil { - return err - } - - } - } - s.assembleMenus() - s.refreshPageCaches() s.setupSitePages() - } - - if err := h.assignMissingTranslations(); err != nil { - return err + sort.Stable(s.workAllPages) } return nil @@ -269,42 +256,60 @@ func (h *HugoSites) assemble(config *BuildCfg) error { } func (h *HugoSites) render(config *BuildCfg) error { + siteRenderContext := &siteRenderContext{cfg: config, multihost: h.multihost} + if !config.PartialReRender { + h.renderFormats = output.Formats{} for _, s := range h.Sites { s.initRenderFormats() + h.renderFormats = append(h.renderFormats, s.renderFormats...) } } + i := 0 for _, s := range h.Sites { - for i, rf := range s.renderFormats { - for _, s2 := range h.Sites { - // We render site by site, but since the content is lazily rendered - // and a site can "borrow" content from other sites, every site - // needs this set. - s2.rc = &siteRenderingContext{Format: rf} - - isRenderingSite := s == s2 - - if !config.PartialReRender { - if err := s2.preparePagesForRender(isRenderingSite && i == 0); err != nil { - return err + for siteOutIdx, renderFormat := range s.renderFormats { + siteRenderContext.outIdx = siteOutIdx + siteRenderContext.sitesOutIdx = i + i++ + + select { + case <-h.Done(): + return nil + default: + // For the non-renderable pages, we use the content iself as + // template and we may have to re-parse and execute it for + // each output format. + h.TemplateHandler().RebuildClone() + + for _, s2 := range h.Sites { + // We render site by site, but since the content is lazily rendered + // and a site can "borrow" content from other sites, every site + // needs this set. + s2.rc = &siteRenderingContext{Format: renderFormat} + + if !config.PartialReRender { + if err := s2.preparePagesForRender(siteRenderContext.sitesOutIdx); err != nil { + return err + } } } - } - - if !config.SkipRender { - if config.PartialReRender { - if err := s.renderPages(config); err != nil { - return err - } - } else { - if err := s.render(config, i); err != nil { - return err + if !config.SkipRender { + if config.PartialReRender { + if err := s.renderPages(siteRenderContext); err != nil { + return err + } + } else { + if err := s.render(siteRenderContext); err != nil { + return err + } } } } + } + } if !config.SkipRender { diff --git a/hugolib/hugo_sites_build_errors_test.go b/hugolib/hugo_sites_build_errors_test.go index dd80946e8..6fe4901a1 100644 --- a/hugolib/hugo_sites_build_errors_test.go +++ b/hugolib/hugo_sites_build_errors_test.go @@ -7,6 +7,9 @@ import ( "runtime" "strings" "testing" + "time" + + "github.com/fortytw2/leaktest" "github.com/gohugoio/hugo/common/herrors" "github.com/stretchr/testify/require" @@ -20,25 +23,24 @@ type testSiteBuildErrorAsserter struct { func (t testSiteBuildErrorAsserter) getFileError(err error) *herrors.ErrorWithFileContext { t.assert.NotNil(err, t.name) ferr := herrors.UnwrapErrorWithFileContext(err) - t.assert.NotNil(ferr, fmt.Sprintf("[%s] got %T: %+v\n%s", t.name, err, err, trace())) + t.assert.NotNil(ferr, fmt.Sprintf("[%s] got %T: %+v\n%s", t.name, err, err, stackTrace())) return ferr } func (t testSiteBuildErrorAsserter) assertLineNumber(lineNumber int, err error) { fe := t.getFileError(err) - t.assert.Equal(lineNumber, fe.Position().LineNumber, fmt.Sprintf("[%s] got => %s\n%s", t.name, fe, trace())) + t.assert.Equal(lineNumber, fe.Position().LineNumber, fmt.Sprintf("[%s] got => %s\n%s", t.name, fe, stackTrace())) } func (t testSiteBuildErrorAsserter) assertErrorMessage(e1, e2 string) { // The error message will contain filenames with OS slashes. Normalize before compare. e1, e2 = filepath.ToSlash(e1), filepath.ToSlash(e2) - t.assert.Contains(e2, e1, trace()) + t.assert.Contains(e2, e1, stackTrace()) } func TestSiteBuildErrors(t *testing.T) { t.Parallel() - assert := require.New(t) const ( yamlcontent = "yamlcontent" @@ -88,9 +90,9 @@ func TestSiteBuildErrors(t *testing.T) { }, assertCreateError: func(a testSiteBuildErrorAsserter, err error) { fe := a.getFileError(err) - assert.Equal(5, fe.Position().LineNumber) - assert.Equal(1, fe.Position().ColumnNumber) - assert.Equal("go-html-template", fe.ChromaLexer) + a.assert.Equal(5, fe.Position().LineNumber) + a.assert.Equal(1, fe.Position().ColumnNumber) + a.assert.Equal("go-html-template", fe.ChromaLexer) a.assertErrorMessage("\"layouts/_default/single.html:5:1\": parse failed: template: _default/single.html:5: unexpected \"}\" in operand", fe.Error()) }, @@ -103,9 +105,9 @@ func TestSiteBuildErrors(t *testing.T) { }, assertBuildError: func(a testSiteBuildErrorAsserter, err error) { fe := a.getFileError(err) - assert.Equal(5, fe.Position().LineNumber) - assert.Equal(14, fe.Position().ColumnNumber) - assert.Equal("go-html-template", fe.ChromaLexer) + a.assert.Equal(5, fe.Position().LineNumber) + a.assert.Equal(14, fe.Position().ColumnNumber) + a.assert.Equal("go-html-template", fe.ChromaLexer) a.assertErrorMessage("\"layouts/_default/single.html:5:14\": execute of template failed", fe.Error()) }, @@ -118,9 +120,9 @@ func TestSiteBuildErrors(t *testing.T) { }, assertBuildError: func(a testSiteBuildErrorAsserter, err error) { fe := a.getFileError(err) - assert.Equal(5, fe.Position().LineNumber) - assert.Equal(14, fe.Position().ColumnNumber) - assert.Equal("go-html-template", fe.ChromaLexer) + a.assert.Equal(5, fe.Position().LineNumber) + a.assert.Equal(14, fe.Position().ColumnNumber) + a.assert.Equal("go-html-template", fe.ChromaLexer) a.assertErrorMessage("\"layouts/_default/single.html:5:14\": execute of template failed", fe.Error()) }, @@ -143,8 +145,8 @@ func TestSiteBuildErrors(t *testing.T) { }, assertBuildError: func(a testSiteBuildErrorAsserter, err error) { fe := a.getFileError(err) - assert.Equal(7, fe.Position().LineNumber) - assert.Equal("md", fe.ChromaLexer) + a.assert.Equal(7, fe.Position().LineNumber) + a.assert.Equal("md", fe.ChromaLexer) // Make sure that it contains both the content file and template a.assertErrorMessage(`content/myyaml.md:7:10": failed to render shortcode "sc"`, fe.Error()) a.assertErrorMessage(`shortcodes/sc.html:4:22: executing "shortcodes/sc.html" at <.Page.Titles>: can't evaluate`, fe.Error()) @@ -158,10 +160,10 @@ func TestSiteBuildErrors(t *testing.T) { }, assertBuildError: func(a testSiteBuildErrorAsserter, err error) { fe := a.getFileError(err) - assert.Equal(7, fe.Position().LineNumber) - assert.Equal(14, fe.Position().ColumnNumber) - assert.Equal("md", fe.ChromaLexer) - a.assertErrorMessage("\"content/myyaml.md:7:14\": failed to extract shortcode: template for shortcode \"nono\" not found", fe.Error()) + a.assert.Equal(7, fe.Position().LineNumber) + a.assert.Equal(10, fe.Position().ColumnNumber) + a.assert.Equal("md", fe.ChromaLexer) + a.assertErrorMessage(`"content/myyaml.md:7:10": failed to extract shortcode: template for shortcode "nono" not found`, fe.Error()) }, }, { @@ -182,8 +184,8 @@ func TestSiteBuildErrors(t *testing.T) { }, assertBuildError: func(a testSiteBuildErrorAsserter, err error) { fe := a.getFileError(err) - assert.Equal(6, fe.Position().LineNumber) - assert.Equal("toml", fe.ErrorContext.ChromaLexer) + a.assert.Equal(6, fe.Position().LineNumber) + a.assert.Equal("toml", fe.ErrorContext.ChromaLexer) }, }, @@ -196,8 +198,8 @@ func TestSiteBuildErrors(t *testing.T) { assertBuildError: func(a testSiteBuildErrorAsserter, err error) { fe := a.getFileError(err) - assert.Equal(3, fe.Position().LineNumber) - assert.Equal("json", fe.ErrorContext.ChromaLexer) + a.assert.Equal(3, fe.Position().LineNumber) + a.assert.Equal("json", fe.ErrorContext.ChromaLexer) }, }, @@ -210,42 +212,43 @@ func TestSiteBuildErrors(t *testing.T) { }, assertBuildError: func(a testSiteBuildErrorAsserter, err error) { - assert.Error(err) + a.assert.Error(err) // This is fixed in latest Go source if regexp.MustCompile("devel|12").MatchString(runtime.Version()) { fe := a.getFileError(err) - assert.Equal(5, fe.Position().LineNumber) - assert.Equal(21, fe.Position().ColumnNumber) + a.assert.Equal(5, fe.Position().LineNumber) + a.assert.Equal(21, fe.Position().ColumnNumber) } else { - assert.Contains(err.Error(), `execute of template failed: panic in Execute`) + a.assert.Contains(err.Error(), `execute of template failed: panic in Execute`) } }, }, } for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert := require.New(t) + errorAsserter := testSiteBuildErrorAsserter{ + assert: assert, + name: test.name, + } - errorAsserter := testSiteBuildErrorAsserter{ - assert: assert, - name: test.name, - } + b := newTestSitesBuilder(t).WithSimpleConfigFile() - b := newTestSitesBuilder(t).WithSimpleConfigFile() + f := func(fileType, content string) string { + if fileType != test.fileType { + return content + } + return test.fileFixer(content) - f := func(fileType, content string) string { - if fileType != test.fileType { - return content } - return test.fileFixer(content) - } - - b.WithTemplatesAdded("layouts/shortcodes/sc.html", f(shortcode, `SHORTCODE L1 + b.WithTemplatesAdded("layouts/shortcodes/sc.html", f(shortcode, `SHORTCODE L1 SHORTCODE L2 SHORTCODE L3: SHORTCODE L4: {{ .Page.Title }} `)) - b.WithTemplatesAdded("layouts/_default/baseof.html", f(base, `BASEOF L1 + b.WithTemplatesAdded("layouts/_default/baseof.html", f(base, `BASEOF L1 BASEOF L2 BASEOF L3 BASEOF L4{{ if .Title }}{{ end }} @@ -253,7 +256,7 @@ BASEOF L4{{ if .Title }}{{ end }} BASEOF L6 `)) - b.WithTemplatesAdded("layouts/_default/single.html", f(single, `{{ define "main" }} + b.WithTemplatesAdded("layouts/_default/single.html", f(single, `{{ define "main" }} SINGLE L2: SINGLE L3: SINGLE L4: @@ -261,7 +264,7 @@ SINGLE L5: {{ .Title }} {{ .Content }} {{ end }} `)) - b.WithContent("myyaml.md", f(yamlcontent, `--- + b.WithContent("myyaml.md", f(yamlcontent, `--- title: "The YAML" --- @@ -275,7 +278,7 @@ The end. `)) - b.WithContent("mytoml.md", f(tomlcontent, `+++ + b.WithContent("mytoml.md", f(tomlcontent, `+++ title = "The TOML" p1 = "v" p2 = "v" @@ -288,7 +291,7 @@ Some content. `)) - b.WithContent("myjson.md", f(jsoncontent, `{ + b.WithContent("myjson.md", f(jsoncontent, `{ "title": "This is a title", "description": "This is a description." } @@ -298,26 +301,30 @@ Some content. `)) - createErr := b.CreateSitesE() - if test.assertCreateError != nil { - test.assertCreateError(errorAsserter, createErr) - } else { - assert.NoError(createErr) - } - - if createErr == nil { - buildErr := b.BuildE(BuildCfg{}) - if test.assertBuildError != nil { - test.assertBuildError(errorAsserter, buildErr) + createErr := b.CreateSitesE() + if test.assertCreateError != nil { + test.assertCreateError(errorAsserter, createErr) } else { - assert.NoError(buildErr) + assert.NoError(createErr) } - } + + if createErr == nil { + buildErr := b.BuildE(BuildCfg{}) + if test.assertBuildError != nil { + test.assertBuildError(errorAsserter, buildErr) + } else { + assert.NoError(buildErr) + } + } + }) } } // https://github.com/gohugoio/hugo/issues/5375 func TestSiteBuildTimeout(t *testing.T) { + if !isCI() { + defer leaktest.CheckTimeout(t, 10*time.Second)() + } b := newTestSitesBuilder(t) b.WithConfigFile("toml", ` @@ -342,6 +349,6 @@ title: "A page" } - b.CreateSites().Build(BuildCfg{}) + b.CreateSites().BuildFail(BuildCfg{}) } diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 83b96b7f4..236fd11a6 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -1,16 +1,16 @@ package hugolib import ( - "bytes" "fmt" "strings" "testing" - "html/template" "os" "path/filepath" "time" + "github.com/gohugoio/hugo/resources/page" + "github.com/fortytw2/leaktest" "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/helpers" @@ -66,8 +66,8 @@ func doTestMultiSitesMainLangInRoot(t *testing.T, defaultInSubDir bool) { assert.Equal("/blog/en/foo", enSite.PathSpec.RelURL("foo", true)) - doc1en := enSite.RegularPages[0] - doc1fr := frSite.RegularPages[0] + doc1en := enSite.RegularPages()[0] + doc1fr := frSite.RegularPages()[0] enPerm := doc1en.Permalink() enRelPerm := doc1en.RelPermalink() @@ -100,7 +100,7 @@ func doTestMultiSitesMainLangInRoot(t *testing.T, defaultInSubDir bool) { // Check list pages b.AssertFileContent(pathMod("public/fr/sect/index.html"), "List", "Bonjour") b.AssertFileContent("public/en/sect/index.html", "List", "Hello") - b.AssertFileContent(pathMod("public/fr/plaques/frtag1/index.html"), "Taxonomy List", "Bonjour") + b.AssertFileContent(pathMod("public/fr/plaques/FRtag1/index.html"), "Taxonomy List", "Bonjour") b.AssertFileContent("public/en/tags/tag1/index.html", "Taxonomy List", "Hello") // Check sitemaps @@ -126,8 +126,8 @@ func doTestMultiSitesMainLangInRoot(t *testing.T, defaultInSubDir bool) { pathMod(`<atom:link href="http://example.com/blog/fr/sect/index.xml"`)) b.AssertFileContent("public/en/sect/index.xml", `<atom:link href="http://example.com/blog/en/sect/index.xml"`) b.AssertFileContent( - pathMod("public/fr/plaques/frtag1/index.xml"), - pathMod(`<atom:link href="http://example.com/blog/fr/plaques/frtag1/index.xml"`)) + pathMod("public/fr/plaques/FRtag1/index.xml"), + pathMod(`<atom:link href="http://example.com/blog/fr/plaques/FRtag1/index.xml"`)) b.AssertFileContent("public/en/tags/tag1/index.xml", `<atom:link href="http://example.com/blog/en/tags/tag1/index.xml"`) // Check paginators @@ -140,12 +140,12 @@ func doTestMultiSitesMainLangInRoot(t *testing.T, defaultInSubDir bool) { b.AssertFileContent(pathMod("public/fr/sect/page/2/index.html"), "List Page 2", "Bonjour", pathMod("http://example.com/blog/fr/sect/")) b.AssertFileContent("public/en/sect/page/2/index.html", "List Page 2", "Hello", "http://example.com/blog/en/sect/") b.AssertFileContent( - pathMod("public/fr/plaques/frtag1/page/1/index.html"), - pathMod(`refresh" content="0; url=http://example.com/blog/fr/plaques/frtag1/"`)) + pathMod("public/fr/plaques/FRtag1/page/1/index.html"), + pathMod(`refresh" content="0; url=http://example.com/blog/fr/plaques/FRtag1/"`)) b.AssertFileContent("public/en/tags/tag1/page/1/index.html", `refresh" content="0; url=http://example.com/blog/en/tags/tag1/"`) b.AssertFileContent( - pathMod("public/fr/plaques/frtag1/page/2/index.html"), "List Page 2", "Bonjour", - pathMod("http://example.com/blog/fr/plaques/frtag1/")) + pathMod("public/fr/plaques/FRtag1/page/2/index.html"), "List Page 2", "Bonjour", + pathMod("http://example.com/blog/fr/plaques/FRtag1/")) b.AssertFileContent("public/en/tags/tag1/page/2/index.html", "List Page 2", "Hello", "http://example.com/blog/en/tags/tag1/") // nn (Nynorsk) and nb (Bokmål) have custom pagePath: side ("page" in Norwegian) b.AssertFileContent("public/nn/side/1/index.html", `refresh" content="0; url=http://example.com/blog/nn/"`) @@ -183,12 +183,12 @@ p1 = "p1en" assert.Len(sites, 2) nnSite := sites[0] - nnHome := nnSite.getPage(KindHome) + nnHome := nnSite.getPage(page.KindHome) assert.Len(nnHome.AllTranslations(), 2) assert.Len(nnHome.Translations(), 1) assert.True(nnHome.IsTranslated()) - enHome := sites[1].getPage(KindHome) + enHome := sites[1].getPage(page.KindHome) p1, err := enHome.Param("p1") assert.NoError(err) @@ -199,9 +199,7 @@ p1 = "p1en" assert.Equal("p1nn", p1) } -// func TestMultiSitesBuild(t *testing.T) { - t.Parallel() for _, config := range []struct { content string @@ -211,7 +209,11 @@ func TestMultiSitesBuild(t *testing.T) { {multiSiteYAMLConfigTemplate, "yml"}, {multiSiteJSONConfigTemplate, "json"}, } { - doTestMultiSitesBuild(t, config.content, config.suffix) + + t.Run(config.suffix, func(t *testing.T) { + t.Parallel() + doTestMultiSitesBuild(t, config.content, config.suffix) + }) } } @@ -228,64 +230,51 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { // Check site config for _, s := range sites { - require.True(t, s.Info.defaultContentLanguageInSubdir, s.Info.Title) + require.True(t, s.Info.defaultContentLanguageInSubdir, s.Info.title) require.NotNil(t, s.disabledKinds) } gp1 := b.H.GetContentPage(filepath.FromSlash("content/sect/doc1.en.md")) require.NotNil(t, gp1) - require.Equal(t, "doc1", gp1.title) + require.Equal(t, "doc1", gp1.Title()) gp2 := b.H.GetContentPage(filepath.FromSlash("content/dummysect/notfound.md")) require.Nil(t, gp2) enSite := sites[0] - enSiteHome := enSite.getPage(KindHome) + enSiteHome := enSite.getPage(page.KindHome) require.True(t, enSiteHome.IsTranslated()) - require.Equal(t, "en", enSite.Language.Lang) - - assert.Equal(5, len(enSite.RegularPages)) - assert.Equal(32, len(enSite.AllPages)) + require.Equal(t, "en", enSite.language.Lang) - doc1en := enSite.RegularPages[0] - permalink := doc1en.Permalink() - require.Equal(t, "http://example.com/blog/en/sect/doc1-slug/", permalink, "invalid doc1.en permalink") - require.Len(t, doc1en.Translations(), 1, "doc1-en should have one translation, excluding itself") + assert.Equal(5, len(enSite.RegularPages())) + assert.Equal(32, len(enSite.AllPages())) - doc2 := enSite.RegularPages[1] - permalink = doc2.Permalink() - require.Equal(t, "http://example.com/blog/en/sect/doc2/", permalink, "invalid doc2 permalink") + // Check 404s + b.AssertFileContent("public/en/404.html", "404|en|404 Page not found") + b.AssertFileContent("public/fr/404.html", "404|fr|404 Page not found") - doc3 := enSite.RegularPages[2] - permalink = doc3.Permalink() - // Note that /superbob is a custom URL set in frontmatter. - // We respect that URL literally (it can be /search.json) - // and do no not do any language code prefixing. - require.Equal(t, "http://example.com/blog/superbob/", permalink, "invalid doc3 permalink") + // Check robots.txt + b.AssertFileContent("public/en/robots.txt", "robots|en|") + b.AssertFileContent("public/nn/robots.txt", "robots|nn|") - require.Equal(t, "/superbob", doc3.URL(), "invalid url, was specified on doc3") - b.AssertFileContent("public/superbob/index.html", "doc3|Hello|en") - require.Equal(t, doc2.PrevPage, doc3, "doc3 should follow doc2, in .PrevPage") + b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Permalink: http://example.com/blog/en/sect/doc1-slug/") + b.AssertFileContent("public/en/sect/doc2/index.html", "Permalink: http://example.com/blog/en/sect/doc2/") + b.AssertFileContent("public/superbob/index.html", "Permalink: http://example.com/blog/superbob/") + doc2 := enSite.RegularPages()[1] + doc3 := enSite.RegularPages()[2] + require.Equal(t, doc2.Prev(), doc3, "doc3 should follow doc2, in .PrevPage") + doc1en := enSite.RegularPages()[0] doc1fr := doc1en.Translations()[0] - permalink = doc1fr.Permalink() - require.Equal(t, "http://example.com/blog/fr/sect/doc1/", permalink, "invalid doc1fr permalink") + b.AssertFileContent("public/fr/sect/doc1/index.html", "Permalink: http://example.com/blog/fr/sect/doc1/") require.Equal(t, doc1en.Translations()[0], doc1fr, "doc1-en should have doc1-fr as translation") require.Equal(t, doc1fr.Translations()[0], doc1en, "doc1-fr should have doc1-en as translation") require.Equal(t, "fr", doc1fr.Language().Lang) - doc4 := enSite.AllPages[4] - permalink = doc4.Permalink() - require.Equal(t, "http://example.com/blog/fr/sect/doc4/", permalink, "invalid doc4 permalink") - require.Equal(t, "/blog/fr/sect/doc4/", doc4.URL()) - + doc4 := enSite.AllPages()[4] require.Len(t, doc4.Translations(), 0, "found translations for doc4") - doc5 := enSite.AllPages[5] - permalink = doc5.Permalink() - require.Equal(t, "http://example.com/blog/fr/somewhere/else/doc5/", permalink, "invalid doc5 permalink") - // Taxonomies and their URLs require.Len(t, enSite.Taxonomies, 1, "should have 1 taxonomy") tags := enSite.Taxonomies["tags"] @@ -294,12 +283,13 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { frSite := sites[1] - require.Equal(t, "fr", frSite.Language.Lang) - require.Len(t, frSite.RegularPages, 4, "should have 3 pages") - require.Len(t, frSite.AllPages, 32, "should have 32 total pages (including translations and nodes)") + require.Equal(t, "fr", frSite.language.Lang) + require.Len(t, frSite.RegularPages(), 4, "should have 3 pages") + require.Len(t, frSite.AllPages(), 32, "should have 32 total pages (including translations and nodes)") - for _, frenchPage := range frSite.RegularPages { - require.Equal(t, "fr", frenchPage.Lang()) + for _, frenchPage := range frSite.RegularPages() { + p := frenchPage + require.Equal(t, "fr", p.Language().Lang) } // See https://github.com/gohugoio/hugo/issues/4285 @@ -307,10 +297,10 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { // isn't ideal in a multilingual setup. You want a way to get the current language version if available. // Now you can do lookups with translation base name to get that behaviour. // Let us test all the regular page variants: - getPageDoc1En := enSite.getPage(KindPage, filepath.ToSlash(doc1en.Path())) - getPageDoc1EnBase := enSite.getPage(KindPage, "sect/doc1") - getPageDoc1Fr := frSite.getPage(KindPage, filepath.ToSlash(doc1fr.Path())) - getPageDoc1FrBase := frSite.getPage(KindPage, "sect/doc1") + getPageDoc1En := enSite.getPage(page.KindPage, filepath.ToSlash(doc1en.File().Path())) + getPageDoc1EnBase := enSite.getPage(page.KindPage, "sect/doc1") + getPageDoc1Fr := frSite.getPage(page.KindPage, filepath.ToSlash(doc1fr.File().Path())) + getPageDoc1FrBase := frSite.getPage(page.KindPage, "sect/doc1") require.Equal(t, doc1en, getPageDoc1En) require.Equal(t, doc1fr, getPageDoc1Fr) require.Equal(t, doc1en, getPageDoc1EnBase) @@ -328,35 +318,36 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Single", "Shortcode: Hello", "LingoDefault") // Check node translations - homeEn := enSite.getPage(KindHome) + homeEn := enSite.getPage(page.KindHome) require.NotNil(t, homeEn) require.Len(t, homeEn.Translations(), 3) - require.Equal(t, "fr", homeEn.Translations()[0].Lang()) - require.Equal(t, "nn", homeEn.Translations()[1].Lang()) - require.Equal(t, "På nynorsk", homeEn.Translations()[1].title) - require.Equal(t, "nb", homeEn.Translations()[2].Lang()) - require.Equal(t, "På bokmål", homeEn.Translations()[2].title, configSuffix) + require.Equal(t, "fr", homeEn.Translations()[0].Language().Lang) + require.Equal(t, "nn", homeEn.Translations()[1].Language().Lang) + require.Equal(t, "På nynorsk", homeEn.Translations()[1].Title()) + require.Equal(t, "nb", homeEn.Translations()[2].Language().Lang) + require.Equal(t, "På bokmål", homeEn.Translations()[2].Title(), configSuffix) require.Equal(t, "Bokmål", homeEn.Translations()[2].Language().LanguageName, configSuffix) - sectFr := frSite.getPage(KindSection, "sect") + sectFr := frSite.getPage(page.KindSection, "sect") require.NotNil(t, sectFr) - require.Equal(t, "fr", sectFr.Lang()) + require.Equal(t, "fr", sectFr.Language().Lang) require.Len(t, sectFr.Translations(), 1) - require.Equal(t, "en", sectFr.Translations()[0].Lang()) - require.Equal(t, "Sects", sectFr.Translations()[0].title) + require.Equal(t, "en", sectFr.Translations()[0].Language().Lang) + require.Equal(t, "Sects", sectFr.Translations()[0].Title()) nnSite := sites[2] - require.Equal(t, "nn", nnSite.Language.Lang) - taxNn := nnSite.getPage(KindTaxonomyTerm, "lag") + require.Equal(t, "nn", nnSite.language.Lang) + taxNn := nnSite.getPage(page.KindTaxonomyTerm, "lag") require.NotNil(t, taxNn) require.Len(t, taxNn.Translations(), 1) - require.Equal(t, "nb", taxNn.Translations()[0].Lang()) + require.Equal(t, "nb", taxNn.Translations()[0].Language().Lang) - taxTermNn := nnSite.getPage(KindTaxonomy, "lag", "sogndal") + taxTermNn := nnSite.getPage(page.KindTaxonomy, "lag", "sogndal") require.NotNil(t, taxTermNn) + require.Equal(t, taxTermNn, nnSite.getPage(page.KindTaxonomy, "LAG", "SOGNDAL")) require.Len(t, taxTermNn.Translations(), 1) - require.Equal(t, "nb", taxTermNn.Translations()[0].Lang()) + require.Equal(t, "nb", taxTermNn.Translations()[0].Language().Lang) // Check sitemap(s) b.AssertFileContent("public/sitemap.xml", @@ -371,59 +362,53 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) { require.Len(t, enTags, 2, fmt.Sprintf("Tags in en: %v", enTags)) require.Len(t, frTags, 2, fmt.Sprintf("Tags in fr: %v", frTags)) require.NotNil(t, enTags["tag1"]) - require.NotNil(t, frTags["frtag1"]) - b.AssertFileContent("public/fr/plaques/frtag1/index.html", "Frtag1|Bonjour|http://example.com/blog/fr/plaques/frtag1/") - b.AssertFileContent("public/en/tags/tag1/index.html", "Tag1|Hello|http://example.com/blog/en/tags/tag1/") + require.NotNil(t, frTags["FRtag1"]) + b.AssertFileContent("public/fr/plaques/FRtag1/index.html", "FRtag1|Bonjour|http://example.com/blog/fr/plaques/FRtag1/") + b.AssertFileContent("public/en/tags/tag1/index.html", "tag1|Hello|http://example.com/blog/en/tags/tag1/") // Check Blackfriday config - require.True(t, strings.Contains(string(doc1fr.content()), "«"), string(doc1fr.content())) - require.False(t, strings.Contains(string(doc1en.content()), "«"), string(doc1en.content())) - require.True(t, strings.Contains(string(doc1en.content()), "“"), string(doc1en.content())) - - // Check that the drafts etc. are not built/processed/rendered. - assertShouldNotBuild(t, b.H) + require.True(t, strings.Contains(content(doc1fr), "«"), content(doc1fr)) + require.False(t, strings.Contains(content(doc1en), "«"), content(doc1en)) + require.True(t, strings.Contains(content(doc1en), "“"), content(doc1en)) // en and nn have custom site menus - require.Len(t, frSite.Menus, 0, "fr: "+configSuffix) - require.Len(t, enSite.Menus, 1, "en: "+configSuffix) - require.Len(t, nnSite.Menus, 1, "nn: "+configSuffix) + require.Len(t, frSite.Menus(), 0, "fr: "+configSuffix) + require.Len(t, enSite.Menus(), 1, "en: "+configSuffix) + require.Len(t, nnSite.Menus(), 1, "nn: "+configSuffix) - require.Equal(t, "Home", enSite.Menus["main"].ByName()[0].Name) - require.Equal(t, "Heim", nnSite.Menus["main"].ByName()[0].Name) - - // Issue #1302 - require.Equal(t, template.URL(""), enSite.RegularPages[0].RSSLink()) + require.Equal(t, "Home", enSite.Menus()["main"].ByName()[0].Name) + require.Equal(t, "Heim", nnSite.Menus()["main"].ByName()[0].Name) // Issue #3108 - prevPage := enSite.RegularPages[0].PrevPage + prevPage := enSite.RegularPages()[0].Prev() require.NotNil(t, prevPage) - require.Equal(t, KindPage, prevPage.Kind) + require.Equal(t, page.KindPage, prevPage.Kind()) for { if prevPage == nil { break } - require.Equal(t, KindPage, prevPage.Kind) - prevPage = prevPage.PrevPage + require.Equal(t, page.KindPage, prevPage.Kind()) + prevPage = prevPage.Prev() } // Check bundles - bundleFr := frSite.getPage(KindPage, "bundles/b1/index.md") + b.AssertFileContent("public/fr/bundles/b1/index.html", "RelPermalink: /blog/fr/bundles/b1/|") + bundleFr := frSite.getPage(page.KindPage, "bundles/b1/index.md") require.NotNil(t, bundleFr) - require.Equal(t, "/blog/fr/bundles/b1/", bundleFr.RelPermalink()) - require.Equal(t, 1, len(bundleFr.Resources)) - logoFr := bundleFr.Resources.GetMatch("logo*") + require.Equal(t, 1, len(bundleFr.Resources())) + logoFr := bundleFr.Resources().GetMatch("logo*") require.NotNil(t, logoFr) - require.Equal(t, "/blog/fr/bundles/b1/logo.png", logoFr.RelPermalink()) + b.AssertFileContent("public/fr/bundles/b1/index.html", "Resources: image/png: /blog/fr/bundles/b1/logo.png") b.AssertFileContent("public/fr/bundles/b1/logo.png", "PNG Data") - bundleEn := enSite.getPage(KindPage, "bundles/b1/index.en.md") + bundleEn := enSite.getPage(page.KindPage, "bundles/b1/index.en.md") require.NotNil(t, bundleEn) - require.Equal(t, "/blog/en/bundles/b1/", bundleEn.RelPermalink()) - require.Equal(t, 1, len(bundleEn.Resources)) - logoEn := bundleEn.Resources.GetMatch("logo*") + b.AssertFileContent("public/en/bundles/b1/index.html", "RelPermalink: /blog/en/bundles/b1/|") + require.Equal(t, 1, len(bundleEn.Resources())) + logoEn := bundleEn.Resources().GetMatch("logo*") require.NotNil(t, logoEn) - require.Equal(t, "/blog/en/bundles/b1/logo.png", logoEn.RelPermalink()) + b.AssertFileContent("public/en/bundles/b1/index.html", "Resources: image/png: /blog/en/bundles/b1/logo.png") b.AssertFileContent("public/en/bundles/b1/logo.png", "PNG Data") } @@ -442,13 +427,13 @@ func TestMultiSitesRebuild(t *testing.T) { sites := b.H.Sites fs := b.Fs - b.AssertFileContent("public/en/sect/doc2/index.html", "Single: doc2|Hello|en|\n\n<h1 id=\"doc2\">doc2</h1>\n\n<p><em>some content</em>") + b.AssertFileContent("public/en/sect/doc2/index.html", "Single: doc2|Hello|en|", "\n\n<h1 id=\"doc2\">doc2</h1>\n\n<p><em>some content</em>") enSite := sites[0] frSite := sites[1] - assert.Len(enSite.RegularPages, 5) - assert.Len(frSite.RegularPages, 4) + assert.Len(enSite.RegularPages(), 5) + assert.Len(frSite.RegularPages(), 4) // Verify translations b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Hello") @@ -458,6 +443,10 @@ func TestMultiSitesRebuild(t *testing.T) { b.AssertFileContent("public/fr/sect/doc1/index.html", "Single", "Shortcode: Bonjour") b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Single", "Shortcode: Hello") + homeEn := enSite.getPage(page.KindHome) + require.NotNil(t, homeEn) + assert.Len(homeEn.Translations(), 3) + contentFs := b.H.BaseFs.Content.Fs for i, this := range []struct { @@ -478,15 +467,15 @@ func TestMultiSitesRebuild(t *testing.T) { }, []fsnotify.Event{{Name: filepath.FromSlash("content/sect/doc2.en.md"), Op: fsnotify.Remove}}, func(t *testing.T) { - assert.Len(enSite.RegularPages, 4, "1 en removed") + assert.Len(enSite.RegularPages(), 4, "1 en removed") // Check build stats - require.Equal(t, 1, enSite.draftCount, "Draft") - require.Equal(t, 1, enSite.futureCount, "Future") - require.Equal(t, 1, enSite.expiredCount, "Expired") - require.Equal(t, 0, frSite.draftCount, "Draft") - require.Equal(t, 1, frSite.futureCount, "Future") - require.Equal(t, 1, frSite.expiredCount, "Expired") + require.Equal(t, 1, enSite.buildStats.draftCount, "Draft") + require.Equal(t, 1, enSite.buildStats.futureCount, "Future") + require.Equal(t, 1, enSite.buildStats.expiredCount, "Expired") + require.Equal(t, 0, frSite.buildStats.draftCount, "Draft") + require.Equal(t, 1, frSite.buildStats.futureCount, "Future") + require.Equal(t, 1, frSite.buildStats.expiredCount, "Expired") }, }, { @@ -501,12 +490,12 @@ func TestMultiSitesRebuild(t *testing.T) { {Name: filepath.FromSlash("content/new1.fr.md"), Op: fsnotify.Create}, }, func(t *testing.T) { - assert.Len(enSite.RegularPages, 6) - assert.Len(enSite.AllPages, 34) - assert.Len(frSite.RegularPages, 5) - require.Equal(t, "new_fr_1", frSite.RegularPages[3].title) - require.Equal(t, "new_en_2", enSite.RegularPages[0].title) - require.Equal(t, "new_en_1", enSite.RegularPages[1].title) + assert.Len(enSite.RegularPages(), 6) + assert.Len(enSite.AllPages(), 34) + assert.Len(frSite.RegularPages(), 5) + require.Equal(t, "new_fr_1", frSite.RegularPages()[3].Title()) + require.Equal(t, "new_en_2", enSite.RegularPages()[0].Title()) + require.Equal(t, "new_en_1", enSite.RegularPages()[1].Title()) rendered := readDestination(t, fs, "public/en/new1/index.html") require.True(t, strings.Contains(rendered, "new_en_1"), rendered) @@ -521,7 +510,7 @@ func TestMultiSitesRebuild(t *testing.T) { }, []fsnotify.Event{{Name: filepath.FromSlash("content/sect/doc1.en.md"), Op: fsnotify.Write}}, func(t *testing.T) { - assert.Len(enSite.RegularPages, 6) + assert.Len(enSite.RegularPages(), 6) doc1 := readDestination(t, fs, "public/en/sect/doc1-slug/index.html") require.True(t, strings.Contains(doc1, "CHANGED"), doc1) @@ -539,8 +528,8 @@ func TestMultiSitesRebuild(t *testing.T) { {Name: filepath.FromSlash("content/new1.en.md"), Op: fsnotify.Rename}, }, func(t *testing.T) { - assert.Len(enSite.RegularPages, 6, "Rename") - require.Equal(t, "new_en_1", enSite.RegularPages[1].title) + assert.Len(enSite.RegularPages(), 6, "Rename") + require.Equal(t, "new_en_1", enSite.RegularPages()[1].Title()) rendered := readDestination(t, fs, "public/en/new1renamed/index.html") require.True(t, strings.Contains(rendered, "new_en_1"), rendered) }}, @@ -554,9 +543,9 @@ func TestMultiSitesRebuild(t *testing.T) { }, []fsnotify.Event{{Name: filepath.FromSlash("layouts/_default/single.html"), Op: fsnotify.Write}}, func(t *testing.T) { - assert.Len(enSite.RegularPages, 6) - assert.Len(enSite.AllPages, 34) - assert.Len(frSite.RegularPages, 5) + assert.Len(enSite.RegularPages(), 6) + assert.Len(enSite.AllPages(), 34) + assert.Len(frSite.RegularPages(), 5) doc1 := readDestination(t, fs, "public/en/sect/doc1-slug/index.html") require.True(t, strings.Contains(doc1, "Template Changed"), doc1) }, @@ -571,18 +560,18 @@ func TestMultiSitesRebuild(t *testing.T) { }, []fsnotify.Event{{Name: filepath.FromSlash("i18n/fr.yaml"), Op: fsnotify.Write}}, func(t *testing.T) { - assert.Len(enSite.RegularPages, 6) - assert.Len(enSite.AllPages, 34) - assert.Len(frSite.RegularPages, 5) + assert.Len(enSite.RegularPages(), 6) + assert.Len(enSite.AllPages(), 34) + assert.Len(frSite.RegularPages(), 5) docEn := readDestination(t, fs, "public/en/sect/doc1-slug/index.html") require.True(t, strings.Contains(docEn, "Hello"), "No Hello") docFr := readDestination(t, fs, "public/fr/sect/doc1/index.html") require.True(t, strings.Contains(docFr, "Salut"), "No Salut") - homeEn := enSite.getPage(KindHome) + homeEn := enSite.getPage(page.KindHome) require.NotNil(t, homeEn) assert.Len(homeEn.Translations(), 3) - require.Equal(t, "fr", homeEn.Translations()[0].Lang()) + require.Equal(t, "fr", homeEn.Translations()[0].Language().Lang) }, }, @@ -595,9 +584,9 @@ func TestMultiSitesRebuild(t *testing.T) { {Name: filepath.FromSlash("layouts/shortcodes/shortcode.html"), Op: fsnotify.Write}, }, func(t *testing.T) { - assert.Len(enSite.RegularPages, 6) - assert.Len(enSite.AllPages, 34) - assert.Len(frSite.RegularPages, 5) + assert.Len(enSite.RegularPages(), 6) + assert.Len(enSite.AllPages(), 34) + assert.Len(frSite.RegularPages(), 5) b.AssertFileContent("public/fr/sect/doc1/index.html", "Single", "Modified Shortcode: Salut") b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Single", "Modified Shortcode: Hello") }, @@ -617,23 +606,6 @@ func TestMultiSitesRebuild(t *testing.T) { this.assertFunc(t) } - // Check that the drafts etc. are not built/processed/rendered. - assertShouldNotBuild(t, b.H) - -} - -func assertShouldNotBuild(t *testing.T, sites *HugoSites) { - s := sites.Sites[0] - - for _, p := range s.rawAllPages { - // No HTML when not processed - require.Equal(t, p.shouldBuild(), bytes.Contains(p.workContent, []byte("</")), p.BaseFileName()+": "+string(p.workContent)) - - require.Equal(t, p.shouldBuild(), p.content() != "", fmt.Sprintf("%v:%v", p.content(), p.shouldBuild())) - - require.Equal(t, p.shouldBuild(), p.content() != "", p.BaseFileName()) - - } } func TestAddNewLanguage(t *testing.T) { @@ -671,31 +643,32 @@ title = "Svenska" enSite := sites.Sites[0] svSite := sites.Sites[1] frSite := sites.Sites[2] - require.True(t, enSite.Language.Lang == "en", enSite.Language.Lang) - require.True(t, svSite.Language.Lang == "sv", svSite.Language.Lang) - require.True(t, frSite.Language.Lang == "fr", frSite.Language.Lang) + require.True(t, enSite.language.Lang == "en", enSite.language.Lang) + require.True(t, svSite.language.Lang == "sv", svSite.language.Lang) + require.True(t, frSite.language.Lang == "fr", frSite.language.Lang) - homeEn := enSite.getPage(KindHome) + homeEn := enSite.getPage(page.KindHome) require.NotNil(t, homeEn) require.Len(t, homeEn.Translations(), 4) - require.Equal(t, "sv", homeEn.Translations()[0].Lang()) - require.Len(t, enSite.RegularPages, 5) - require.Len(t, frSite.RegularPages, 4) + require.Equal(t, "sv", homeEn.Translations()[0].Language().Lang) + + require.Len(t, enSite.RegularPages(), 5) + require.Len(t, frSite.RegularPages(), 4) // Veriy Swedish site - require.Len(t, svSite.RegularPages, 1) - svPage := svSite.RegularPages[0] + require.Len(t, svSite.RegularPages(), 1) + svPage := svSite.RegularPages()[0] - require.Equal(t, "Swedish Contentfile", svPage.title) - require.Equal(t, "sv", svPage.Lang()) + require.Equal(t, "Swedish Contentfile", svPage.Title()) + require.Equal(t, "sv", svPage.Language().Lang) require.Len(t, svPage.Translations(), 2) require.Len(t, svPage.AllTranslations(), 3) - require.Equal(t, "en", svPage.Translations()[0].Lang()) + require.Equal(t, "en", svPage.Translations()[0].Language().Lang) // Regular pages have no children - require.Len(t, svPage.Pages, 0) - require.Len(t, svPage.data["Pages"], 0) + require.Len(t, svPage.Pages(), 0) + require.Len(t, svPage.Data().(page.Data).Pages(), 0) } @@ -782,12 +755,12 @@ Some text. Some more text. content = append(content, []string{"s2/_index.md", fmt.Sprintf(contentTempl, defaultOutputs, fmt.Sprintf("S %d", 2), 2, true)}...) b.WithSimpleConfigFile() - b.WithTemplates("layouts/_default/single.html", `Single: {{ .Content }}`) + b.WithTemplates("layouts/_default/single.html", `Single: {{ .Content }}|RelPermalink: {{ .RelPermalink }}|Permalink: {{ .Permalink }}`) b.WithTemplates("layouts/_default/myview.html", `View: {{ len .Content }}`) - b.WithTemplates("layouts/_default/single.json", `Single JSON: {{ .Content }}`) + b.WithTemplates("layouts/_default/single.json", `Single JSON: {{ .Content }}|RelPermalink: {{ .RelPermalink }}|Permalink: {{ .Permalink }}`) b.WithTemplates("layouts/_default/list.html", ` Page: {{ .Paginator.PageNumber }} -P: {{ path.Join .Path }} +P: {{ with .File }}{{ path.Join .Path }}{{ end }} List: {{ len .Paginator.Pages }}|List Content: {{ len .Content }} {{ $shuffled := where .Site.RegularPages "Params.multioutput" true | shuffle }} {{ $first5 := $shuffled | first 5 }} @@ -810,7 +783,7 @@ END if i%10 == 0 { section = "s2" } - checkContent(b, fmt.Sprintf("public/%s/page%d/index.html", section, i), 8343, contentMatchers...) + checkContent(b, fmt.Sprintf("public/%s/page%d/index.html", section, i), contentMatchers...) } } @@ -819,48 +792,158 @@ END if i%10 == 0 { section = "s2" } - checkContent(b, fmt.Sprintf("public/%s/page%d/index.json", section, i), 8348, contentMatchers...) + checkContent(b, fmt.Sprintf("public/%s/page%d/index.json", section, i), contentMatchers...) } - checkContent(b, "public/s1/index.html", 184, "P: s1/_index.md\nList: 10|List Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335\n\nRender 1: View: 8335\n\nRender 2: View: 8335\n\nRender 3: View: 8335\n\nRender 4: View: 8335\n\nEND\n") - checkContent(b, "public/s2/index.html", 184, "P: s2/_index.md\nList: 10|List Content: 8335", "Render 4: View: 8335\n\nEND") - checkContent(b, "public/index.html", 181, "P: _index.md\nList: 10|List Content: 8335", "4: View: 8335\n\nEND") + checkContent(b, "public/s1/index.html", "P: s1/_index.md\nList: 10|List Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335\n\nRender 1: View: 8335\n\nRender 2: View: 8335\n\nRender 3: View: 8335\n\nRender 4: View: 8335\n\nEND\n") + checkContent(b, "public/s2/index.html", "P: s2/_index.md\nList: 10|List Content: 8335", "Render 4: View: 8335\n\nEND") + checkContent(b, "public/index.html", "P: _index.md\nList: 10|List Content: 8335", "4: View: 8335\n\nEND") - // Chek paginated pages + // Check paginated pages for i := 2; i <= 9; i++ { - checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), 181, fmt.Sprintf("Page: %d", i), "Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335", "Render 4: View: 8335\n\nEND") + checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), fmt.Sprintf("Page: %d", i), "Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335", "Render 4: View: 8335\n\nEND") } } -func checkContent(s *sitesBuilder, filename string, length int, matches ...string) { +func checkContent(s *sitesBuilder, filename string, matches ...string) { content := readDestination(s.T, s.Fs, filename) for _, match := range matches { if !strings.Contains(content, match) { s.Fatalf("No match for %q in content for %s\n%q", match, filename, content) } } - if len(content) != length { - s.Fatalf("got %d expected %d", len(content), length) + +} + +func TestTranslationsFromContentToNonContent(t *testing.T) { + b := newTestSitesBuilder(t) + b.WithConfigFile("toml", ` + +baseURL = "http://example.com/" + +defaultContentLanguage = "en" + +[languages] +[languages.en] +weight = 10 +contentDir = "content/en" +[languages.nn] +weight = 20 +contentDir = "content/nn" + + +`) + + b.WithContent("en/mysection/_index.md", ` +--- +Title: My Section +--- + +`) + + b.WithContent("en/_index.md", ` +--- +Title: My Home +--- + +`) + + b.WithContent("en/categories/mycat/_index.md", ` +--- +Title: My MyCat +--- + +`) + + b.WithContent("en/categories/_index.md", ` +--- +Title: My categories +--- + +`) + + for _, lang := range []string{"en", "nn"} { + + b.WithContent(lang+"/mysection/page.md", ` +--- +Title: My Page +categories: ["mycat"] +--- + +`) + + } + + b.Build(BuildCfg{}) + + for _, path := range []string{ + "/", + "/mysection", + "/categories", + "/categories/mycat", + } { + + t.Run(path, func(t *testing.T) { + assert := require.New(t) + + s1, _ := b.H.Sites[0].getPageNew(nil, path) + s2, _ := b.H.Sites[1].getPageNew(nil, path) + + assert.NotNil(s1) + assert.NotNil(s2) + + assert.Equal(1, len(s1.Translations())) + assert.Equal(1, len(s2.Translations())) + assert.Equal(s2, s1.Translations()[0]) + assert.Equal(s1, s2.Translations()[0]) + + m1 := s1.Translations().MergeByLanguage(s2.Translations()) + m2 := s2.Translations().MergeByLanguage(s1.Translations()) + + assert.Equal(1, len(m1)) + assert.Equal(1, len(m2)) + }) + } } +// https://github.com/gohugoio/hugo/issues/5777 func TestTableOfContentsInShortcodes(t *testing.T) { t.Parallel() b := newMultiSiteTestDefaultBuilder(t) b.WithTemplatesAdded("layouts/shortcodes/toc.html", tocShortcode) + b.WithTemplatesAdded("layouts/shortcodes/wrapper.html", "{{ .Inner }}") b.WithContent("post/simple.en.md", tocPageSimple) + b.WithContent("post/variants1.en.md", tocPageVariants1) + b.WithContent("post/variants2.en.md", tocPageVariants2) + b.WithContent("post/withSCInHeading.en.md", tocPageWithShortcodesInHeadings) b.CreateSites().Build(BuildCfg{}) - b.AssertFileContent("public/en/post/simple/index.html", tocPageSimpleExpected) + b.AssertFileContent("public/en/post/simple/index.html", + tocPageSimpleExpected, + // Make sure it is inserted twice + `TOC1: <nav id="TableOfContents">`, + `TOC2: <nav id="TableOfContents">`, + ) + + b.AssertFileContentFn("public/en/post/variants1/index.html", func(s string) bool { + return strings.Count(s, "TableOfContents") == 4 + }) + b.AssertFileContentFn("public/en/post/variants2/index.html", func(s string) bool { + return strings.Count(s, "TableOfContents") == 6 + }) + b.AssertFileContent("public/en/post/withSCInHeading/index.html", tocPageWithShortcodesInHeadingsExpected) } var tocShortcode = ` -{{ .Page.TableOfContents }} +TOC1: {{ .Page.TableOfContents }} + +TOC2: {{ .Page.TableOfContents }} ` func TestSelfReferencedContentInShortcode(t *testing.T) { @@ -901,6 +984,41 @@ Even more text. Lorem ipsum... ` +var tocPageVariants1 = `--- +title: tocTest +publishdate: "2000-01-01" +--- +Variant 1: +{{% wrapper %}} +{{< toc >}} +{{% /wrapper %}} +# Heading 1 + +Variant 3: +{{% toc %}} + +` + +var tocPageVariants2 = `--- +title: tocTest +publishdate: "2000-01-01" +--- +Variant 1: +{{% wrapper %}} +{{< toc >}} +{{% /wrapper %}} +# Heading 1 + +Variant 2: +{{< wrapper >}} +{{< toc >}} +{{< /wrapper >}} + +Variant 3: +{{% toc %}} + +` + var tocPageSimpleExpected = `<nav id="TableOfContents"> <ul> <li><a href="#1">Heading 1</a> @@ -958,6 +1076,7 @@ paginate = 1 disablePathToLower = true defaultContentLanguage = "{{ .DefaultContentLanguage }}" defaultContentLanguageInSubdir = {{ .DefaultContentLanguageInSubdir }} +enableRobotsTXT = true [permalinks] other = "/somewhere/else/:filename" @@ -1015,6 +1134,7 @@ disablePathToLower: true paginate: 1 defaultContentLanguage: "{{ .DefaultContentLanguage }}" defaultContentLanguageInSubdir: {{ .DefaultContentLanguageInSubdir }} +enableRobotsTXT: true permalinks: other: "/somewhere/else/:filename" @@ -1073,6 +1193,7 @@ var multiSiteJSONConfigTemplate = ` "disablePathToLower": true, "defaultContentLanguage": "{{ .DefaultContentLanguage }}", "defaultContentLanguageInSubdir": true, + "enableRobotsTXT": true, "permalinks": { "other": "/somewhere/else/:filename" }, @@ -1170,7 +1291,23 @@ func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { b, err := afero.ReadFile(fs, filename) if err != nil { // Print some debug info - root := strings.Split(filename, helpers.FilePathSeparator)[0] + hadSlash := strings.HasPrefix(filename, helpers.FilePathSeparator) + start := 0 + if hadSlash { + start = 1 + } + end := start + 1 + + parts := strings.Split(filename, helpers.FilePathSeparator) + if parts[start] == "work" { + end++ + } + + root := filepath.Join(parts[start:end]...) + if hadSlash { + root = helpers.FilePathSeparator + root + } + helpers.PrintFs(fs, root, os.Stdout) Fatalf(t, "Failed to read file: %s", err) } @@ -1262,8 +1399,8 @@ NOTE: slug should be used as URL title: doc1 weight: 1 plaques: - - frtag1 - - frtag2 + - FRtag1 + - FRtag2 publishdate: "2000-01-04" --- # doc1 @@ -1293,7 +1430,7 @@ aliases: [/en/al/alias1,/al/alias2/] tags: - tag2 - tag1 -url: /superbob +url: /superbob/ --- # doc3 *some content* @@ -1303,7 +1440,7 @@ NOTE: third 'en' doc, should trigger pagination on home page. title: doc4 weight: 4 plaques: - - frtag1 + - FRtag1 publishdate: "2000-01-05" --- # doc4 diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go index 83d6bfc9e..999d94559 100644 --- a/hugolib/hugo_sites_multihost_test.go +++ b/hugolib/hugo_sites_multihost_test.go @@ -3,6 +3,8 @@ package hugolib import ( "testing" + "github.com/gohugoio/hugo/resources/page" + "github.com/stretchr/testify/require" ) @@ -55,7 +57,7 @@ languageName = "Nynorsk" s1 := b.H.Sites[0] - s1h := s1.getPage(KindHome) + s1h := s1.getPage(page.KindHome) assert.True(s1h.IsTranslated()) assert.Len(s1h.Translations(), 2) assert.Equal("https://example.com/docs/", s1h.Permalink()) @@ -66,9 +68,8 @@ languageName = "Nynorsk" // For multihost, we never want any content in the root. // // check url in front matter: - pageWithURLInFrontMatter := s1.getPage(KindPage, "sect/doc3.en.md") + pageWithURLInFrontMatter := s1.getPage(page.KindPage, "sect/doc3.en.md") assert.NotNil(pageWithURLInFrontMatter) - assert.Equal("/superbob", pageWithURLInFrontMatter.URL()) assert.Equal("/docs/superbob/", pageWithURLInFrontMatter.RelPermalink()) b.AssertFileContent("public/en/superbob/index.html", "doc3|Hello|en") @@ -78,7 +79,7 @@ languageName = "Nynorsk" s2 := b.H.Sites[1] - s2h := s2.getPage(KindHome) + s2h := s2.getPage(page.KindHome) assert.Equal("https://example.fr/", s2h.Permalink()) b.AssertFileContent("public/fr/index.html", "French Home Page", "String Resource: /docs/text/pipes.txt") @@ -94,22 +95,19 @@ languageName = "Nynorsk" // Check bundles - bundleEn := s1.getPage(KindPage, "bundles/b1/index.en.md") + bundleEn := s1.getPage(page.KindPage, "bundles/b1/index.en.md") require.NotNil(t, bundleEn) require.Equal(t, "/docs/bundles/b1/", bundleEn.RelPermalink()) - require.Equal(t, 1, len(bundleEn.Resources)) - logoEn := bundleEn.Resources.GetMatch("logo*") - require.NotNil(t, logoEn) - require.Equal(t, "/docs/bundles/b1/logo.png", logoEn.RelPermalink()) + require.Equal(t, 1, len(bundleEn.Resources())) + b.AssertFileContent("public/en/bundles/b1/logo.png", "PNG Data") + b.AssertFileContent("public/en/bundles/b1/index.html", " image/png: /docs/bundles/b1/logo.png") - bundleFr := s2.getPage(KindPage, "bundles/b1/index.md") + bundleFr := s2.getPage(page.KindPage, "bundles/b1/index.md") require.NotNil(t, bundleFr) require.Equal(t, "/bundles/b1/", bundleFr.RelPermalink()) - require.Equal(t, 1, len(bundleFr.Resources)) - logoFr := bundleFr.Resources.GetMatch("logo*") - require.NotNil(t, logoFr) - require.Equal(t, "/bundles/b1/logo.png", logoFr.RelPermalink()) + require.Equal(t, 1, len(bundleFr.Resources())) b.AssertFileContent("public/fr/bundles/b1/logo.png", "PNG Data") + b.AssertFileContent("public/fr/bundles/b1/index.html", " image/png: /bundles/b1/logo.png") } diff --git a/hugolib/hugo_smoke_test.go b/hugolib/hugo_smoke_test.go new file mode 100644 index 000000000..d5b8861ce --- /dev/null +++ b/hugolib/hugo_smoke_test.go @@ -0,0 +1,303 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSmoke(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + const configFile = ` +baseURL = "https://example.com" +title = "Simple Site" +rssLimit = 3 +defaultContentLanguage = "en" +enableRobotsTXT = true + +[languages] +[languages.en] +weight = 1 +title = "In English" +[languages.no] +weight = 2 +title = "På norsk" + +[params] +hugo = "Rules!" + +[outputs] + home = ["HTML", "JSON", "CSV", "RSS"] + +` + + const pageContentAndSummaryDivider = `--- +title: Page with outputs +hugo: "Rocks!" +outputs: ["HTML", "JSON"] +tags: [ "hugo" ] +aliases: [ "/a/b/c" ] +--- + +This is summary. + +<!--more--> + +This is content with some shortcodes. + +Shortcode 1: {{< sc >}}. +Shortcode 2: {{< sc >}}. + +` + + const pageContentWithMarkdownShortcodes = `--- +title: Page with markdown shortcode +hugo: "Rocks!" +outputs: ["HTML", "JSON"] +--- + +This is summary. + +<!--more--> + +This is content[^a]. + +# Header above + +{{% markdown-shortcode %}} +# Header inside + +Some **markdown**.[^b] + +{{% /markdown-shortcode %}} + +# Heder below + +Some more content[^c]. + +Footnotes: + +[^a]: Fn 1 +[^b]: Fn 2 +[^c]: Fn 3 + +` + + var pageContentAutoSummary = strings.Replace(pageContentAndSummaryDivider, "<!--more-->", "", 1) + + b := newTestSitesBuilder(t).WithConfigFile("toml", configFile) + b.WithTemplatesAdded("shortcodes/markdown-shortcode.html", ` +Some **Markdown** in shortcode. + +{{ .Inner }} + + + +`) + + b.WithTemplatesAdded("shortcodes/markdown-shortcode.json", ` +Some **Markdown** in JSON shortcode. +{{ .Inner }} + +`) + + for i := 1; i <= 11; i++ { + if i%2 == 0 { + b.WithContent(fmt.Sprintf("blog/page%d.md", i), pageContentAndSummaryDivider) + b.WithContent(fmt.Sprintf("blog/page%d.no.md", i), pageContentAndSummaryDivider) + } else { + b.WithContent(fmt.Sprintf("blog/page%d.md", i), pageContentAutoSummary) + } + } + + for i := 1; i <= 5; i++ { + // Root section pages + b.WithContent(fmt.Sprintf("root%d.md", i), pageContentAutoSummary) + } + + // https://github.com/gohugoio/hugo/issues/4695 + b.WithContent("blog/markyshort.md", pageContentWithMarkdownShortcodes) + + // Add one bundle + b.WithContent("blog/mybundle/index.md", pageContentAndSummaryDivider) + b.WithContent("blog/mybundle/mydata.csv", "Bundled CSV") + + const ( + commonPageTemplate = `|{{ .Kind }}|{{ .Title }}|{{ .Path }}|{{ .Summary }}|{{ .Content }}|RelPermalink: {{ .RelPermalink }}|WordCount: {{ .WordCount }}|Pages: {{ .Pages }}|Data Pages: Pages({{ len .Data.Pages }})|Resources: {{ len .Resources }}|Summary: {{ .Summary }}` + commonPaginatorTemplate = `|Paginator: {{ with .Paginator }}{{ .PageNumber }}{{ else }}NIL{{ end }}` + commonListTemplateNoPaginator = `|{{ range $i, $e := (.Pages | first 1) }}|Render {{ $i }}: {{ .Kind }}|{{ .Render "li" }}|{{ end }}|Site params: {{ $.Site.Params.hugo }}|RelPermalink: {{ .RelPermalink }}` + commonListTemplate = commonPaginatorTemplate + `|{{ range $i, $e := (.Pages | first 1) }}|Render {{ $i }}: {{ .Kind }}|{{ .Render "li" }}|{{ end }}|Site params: {{ $.Site.Params.hugo }}|RelPermalink: {{ .RelPermalink }}` + commonShortcodeTemplate = `|{{ .Name }}|{{ .Ordinal }}|{{ .Page.Summary }}|{{ .Page.Content }}|WordCount: {{ .Page.WordCount }}` + prevNextTemplate = `|Prev: {{ with .Prev }}{{ .RelPermalink }}{{ end }}|Next: {{ with .Next }}{{ .RelPermalink }}{{ end }}` + prevNextInSectionTemplate = `|PrevInSection: {{ with .PrevInSection }}{{ .RelPermalink }}{{ end }}|NextInSection: {{ with .NextInSection }}{{ .RelPermalink }}{{ end }}` + paramsTemplate = `|Params: {{ .Params.hugo }}` + treeNavTemplate = `|CurrentSection: {{ .CurrentSection }}` + ) + + b.WithTemplates( + "_default/list.html", "HTML: List"+commonPageTemplate+commonListTemplate+"|First Site: {{ .Sites.First.Title }}", + "_default/list.json", "JSON: List"+commonPageTemplate+commonListTemplateNoPaginator, + "_default/list.csv", "CSV: List"+commonPageTemplate+commonListTemplateNoPaginator, + "_default/single.html", "HTML: Single"+commonPageTemplate+prevNextTemplate+prevNextInSectionTemplate+treeNavTemplate, + "_default/single.json", "JSON: Single"+commonPageTemplate, + + // For .Render test + "_default/li.html", `HTML: LI|{{ strings.Contains .Content "HTML: Shortcode: sc" }}`+paramsTemplate, + "_default/li.json", `JSON: LI|{{ strings.Contains .Content "JSON: Shortcode: sc" }}`+paramsTemplate, + "_default/li.csv", `CSV: LI|{{ strings.Contains .Content "CSV: Shortcode: sc" }}`+paramsTemplate, + + "404.html", "{{ .Kind }}|{{ .Title }}|Page not found", + + "shortcodes/sc.html", "HTML: Shortcode: "+commonShortcodeTemplate, + "shortcodes/sc.json", "JSON: Shortcode: "+commonShortcodeTemplate, + "shortcodes/sc.csv", "CSV: Shortcode: "+commonShortcodeTemplate, + ) + + b.CreateSites().Build(BuildCfg{}) + + b.AssertFileContent("public/blog/page1/index.html", + "This is content with some shortcodes.", + "Page with outputs", + "Pages: Pages(0)", + "RelPermalink: /blog/page1/|", + "Shortcode 1: HTML: Shortcode: |sc|0|||WordCount: 0.", + "Shortcode 2: HTML: Shortcode: |sc|1|||WordCount: 0.", + "Prev: /blog/page10/|Next: /blog/mybundle/", + "PrevInSection: /blog/page10/|NextInSection: /blog/mybundle/", + "Summary: This is summary.", + "CurrentSection: Page(/blog)", + ) + + b.AssertFileContent("public/blog/page1/index.json", + "JSON: Single|page|Page with outputs|", + "SON: Shortcode: |sc|0||") + + b.AssertFileContent("public/index.html", + "home|In English", + "Site params: Rules", + "Pages: Pages(18)|Data Pages: Pages(18)", + "Paginator: 1", + "First Site: In English", + "RelPermalink: /", + ) + + b.AssertFileContent("public/no/index.html", "home|På norsk", "RelPermalink: /no/") + + // Check RSS + rssHome := b.FileContent("public/index.xml") + assert.Contains(rssHome, `<atom:link href="https://example.com/index.xml" rel="self" type="application/rss+xml" />`) + assert.Equal(3, strings.Count(rssHome, "<item>")) // rssLimit = 3 + + // .Render should use template/content from the current output format + // even if that output format isn't configured for that page. + b.AssertFileContent( + "public/index.json", + "Render 0: page|JSON: LI|false|Params: Rocks!", + ) + + b.AssertFileContent( + "public/index.html", + "Render 0: page|HTML: LI|false|Params: Rocks!|", + ) + + b.AssertFileContent( + "public/index.csv", + "Render 0: page|CSV: LI|false|Params: Rocks!|", + ) + + // Check bundled resources + b.AssertFileContent( + "public/blog/mybundle/index.html", + "Resources: 1", + ) + + // Check pages in root section + b.AssertFileContent( + "public/root3/index.html", + "Single|page|Page with outputs|root3.md|", + "Prev: /root4/|Next: /root2/|PrevInSection: /root4/|NextInSection: /root2/", + ) + + b.AssertFileContent( + "public/root3/index.json", "Shortcode 1: JSON:") + + // Paginators + b.AssertFileContent("public/page/1/index.html", `rel="canonical" href="https://example.com/"`) + b.AssertFileContent("public/page/2/index.html", "HTML: List|home|In English|", "Paginator: 2") + + // 404 + b.AssertFileContent("public/404.html", "404|404 Page not found") + + // Sitemaps + b.AssertFileContent("public/en/sitemap.xml", "<loc>https://example.com/blog/</loc>") + b.AssertFileContent("public/no/sitemap.xml", `hreflang="no"`) + + b.AssertFileContent("public/sitemap.xml", "<loc>https://example.com/en/sitemap.xml</loc>", "<loc>https://example.com/no/sitemap.xml</loc>") + + // robots.txt + b.AssertFileContent("public/robots.txt", `User-agent: *`) + + // Aliases + b.AssertFileContent("public/a/b/c/index.html", `refresh`) + + // Markdown vs shortcodes + // Check that all footnotes are grouped (even those from inside the shortcode) + b.AssertFileContentRe("public/blog/markyshort/index.html", `Footnotes:.*<ol>.*Fn 1.*Fn 2.*Fn 3.*</ol>`) + +} + +// https://github.com/golang/go/issues/30286 +func TestDataRace(t *testing.T) { + + const page = ` +--- +title: "The Page" +outputs: ["HTML", "JSON"] +--- + +The content. + + + ` + + b := newTestSitesBuilder(t).WithSimpleConfigFile() + for i := 1; i <= 50; i++ { + b.WithContent(fmt.Sprintf("blog/page%d.md", i), page) + } + + b.WithContent("_index.md", ` +--- +title: "The Home" +outputs: ["HTML", "JSON", "CSV", "RSS"] +--- + +The content. + + +`) + + commonTemplate := `{{ .Data.Pages }}` + + b.WithTemplatesAdded("_default/single.html", "HTML Single: "+commonTemplate) + b.WithTemplatesAdded("_default/list.html", "HTML List: "+commonTemplate) + + b.CreateSites().Build(BuildCfg{}) +} diff --git a/hugolib/language_content_dir_test.go b/hugolib/language_content_dir_test.go index 577fdfaeb..ad1e1fb53 100644 --- a/hugolib/language_content_dir_test.go +++ b/hugolib/language_content_dir_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,6 +19,8 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/resources/page" + "github.com/stretchr/testify/require" ) @@ -99,15 +101,19 @@ Content. section := "sect" var contentRoot = func(lang string) string { - contentRoot := "content/main" - switch lang { case "nn": - contentRoot = "content/norsk" + return "content/norsk" case "sv": - contentRoot = "content/svensk" + return "content/svensk" + default: + return "content/main" } - return contentRoot + "/" + section + + } + + var contentSectionRoot = func(lang string) string { + return contentRoot(lang) + "/" + section } for _, lang := range []string{"en", "nn", "sv"} { @@ -124,7 +130,7 @@ Content. } base := fmt.Sprintf("p-%s-%d", lang, j) - slug := fmt.Sprintf("%s", base) + slug := base langID := "" if lang == "sv" && j%4 == 0 { @@ -139,7 +145,7 @@ Content. slug += langID - contentRoot := contentRoot(lang) + contentRoot := contentSectionRoot(lang) filename := filepath.Join(contentRoot, fmt.Sprintf("page%d%s.md", j, langID)) contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, slug, slug, j)) @@ -148,7 +154,7 @@ Content. // Put common translations in all of them for i, lang := range []string{"en", "nn", "sv"} { - contentRoot := contentRoot(lang) + contentRoot := contentSectionRoot(lang) slug := fmt.Sprintf("common_%s", lang) @@ -173,7 +179,7 @@ Content. // Add a bundle with some images for i, lang := range []string{"en", "nn", "sv"} { - contentRoot := contentRoot(lang) + contentRoot := contentSectionRoot(lang) slug := fmt.Sprintf("bundle_%s", lang) filename := filepath.Join(contentRoot, "mybundle", "index.md") contentFiles = append(contentFiles, filename, fmt.Sprintf(pageBundleTemplate, slug, 400+i)) @@ -190,11 +196,20 @@ Content. } + // Add some static files inside the content dir + // https://github.com/gohugoio/hugo/issues/5759 + for _, lang := range []string{"en", "nn", "sv"} { + contentRoot := contentRoot(lang) + for i := 0; i < 2; i++ { + filename := filepath.Join(contentRoot, "mystatic", fmt.Sprintf("file%d.yaml", i)) + contentFiles = append(contentFiles, filename, lang) + } + } + b := newTestSitesBuilder(t) b.WithWorkingDir("/my/project").WithConfigFile("toml", config).WithContent(contentFiles...).CreateSites() _ = os.Stdout - //printFs(b.H.BaseFs.ContentFs, "/", os.Stdout) b.Build(BuildCfg{}) @@ -204,11 +219,14 @@ Content. nnSite := b.H.Sites[1] svSite := b.H.Sites[2] + b.AssertFileContent("/my/project/public/en/mystatic/file1.yaml", "en") + b.AssertFileContent("/my/project/public/nn/mystatic/file1.yaml", "nn") + //dumpPages(nnSite.RegularPages...) - assert.Equal(12, len(nnSite.RegularPages)) - assert.Equal(13, len(enSite.RegularPages)) + assert.Equal(12, len(nnSite.RegularPages())) + assert.Equal(13, len(enSite.RegularPages())) - assert.Equal(10, len(svSite.RegularPages)) + assert.Equal(10, len(svSite.RegularPages())) svP2, err := svSite.getPageNew(nil, "/sect/page2.md") assert.NoError(err) @@ -217,9 +235,9 @@ Content. enP2, err := enSite.getPageNew(nil, "/sect/page2.md") assert.NoError(err) - assert.Equal("en", enP2.Lang()) - assert.Equal("sv", svP2.Lang()) - assert.Equal("nn", nnP2.Lang()) + assert.Equal("en", enP2.Language().Lang) + assert.Equal("sv", svP2.Language().Lang) + assert.Equal("nn", nnP2.Language().Lang) content, _ := nnP2.Content() assert.Contains(content, "SVP3-REF: https://example.org/sv/sect/p-sv-3/") @@ -241,10 +259,10 @@ Content. assert.NoError(err) assert.Equal("https://example.org/nn/sect/p-nn-3/", nnP3Ref) - for i, p := range enSite.RegularPages { + for i, p := range enSite.RegularPages() { j := i + 1 msg := fmt.Sprintf("Test %d", j) - assert.Equal("en", p.Lang(), msg) + assert.Equal("en", p.Language().Lang, msg) assert.Equal("sect", p.Section()) if j < 9 { if j%4 == 0 { @@ -256,20 +274,20 @@ Content. } // Check bundles - bundleEn := enSite.RegularPages[len(enSite.RegularPages)-1] - bundleNn := nnSite.RegularPages[len(nnSite.RegularPages)-1] - bundleSv := svSite.RegularPages[len(svSite.RegularPages)-1] + bundleEn := enSite.RegularPages()[len(enSite.RegularPages())-1] + bundleNn := nnSite.RegularPages()[len(nnSite.RegularPages())-1] + bundleSv := svSite.RegularPages()[len(svSite.RegularPages())-1] assert.Equal("/en/sect/mybundle/", bundleEn.RelPermalink()) assert.Equal("/sv/sect/mybundle/", bundleSv.RelPermalink()) - assert.Equal(4, len(bundleEn.Resources)) - assert.Equal(4, len(bundleNn.Resources)) - assert.Equal(4, len(bundleSv.Resources)) + assert.Equal(4, len(bundleEn.Resources())) + assert.Equal(4, len(bundleNn.Resources())) + assert.Equal(4, len(bundleSv.Resources())) - assert.Equal("/en/sect/mybundle/logo.png", bundleEn.Resources.GetMatch("logo*").RelPermalink()) - assert.Equal("/nn/sect/mybundle/logo.png", bundleNn.Resources.GetMatch("logo*").RelPermalink()) - assert.Equal("/sv/sect/mybundle/logo.png", bundleSv.Resources.GetMatch("logo*").RelPermalink()) + b.AssertFileContent("/my/project/public/en/sect/mybundle/index.html", "image/png: /en/sect/mybundle/logo.png") + b.AssertFileContent("/my/project/public/nn/sect/mybundle/index.html", "image/png: /nn/sect/mybundle/logo.png") + b.AssertFileContent("/my/project/public/sv/sect/mybundle/index.html", "image/png: /sv/sect/mybundle/logo.png") b.AssertFileContent("/my/project/public/sv/sect/mybundle/featured.png", "PNG Data for sv") b.AssertFileContent("/my/project/public/nn/sect/mybundle/featured.png", "PNG Data for nn") @@ -278,9 +296,9 @@ Content. b.AssertFileContent("/my/project/public/sv/sect/mybundle/logo.png", "PNG Data") b.AssertFileContent("/my/project/public/nn/sect/mybundle/logo.png", "PNG Data") - nnSect := nnSite.getPage(KindSection, "sect") + nnSect := nnSite.getPage(page.KindSection, "sect") assert.NotNil(nnSect) - assert.Equal(12, len(nnSect.Pages)) + assert.Equal(12, len(nnSect.Pages())) nnHome, _ := nnSite.Info.Home() assert.Equal("/nn/", nnHome.RelPermalink()) diff --git a/hugolib/media.go b/hugolib/media.go deleted file mode 100644 index aae9a7870..000000000 --- a/hugolib/media.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -// An Image contains metadata for images + image sitemaps -// https://support.google.com/webmasters/answer/178636?hl=en -type Image struct { - - // The URL of the image. In some cases, the image URL may not be on the - // same domain as your main site. This is fine, as long as both domains - // are verified in Webmaster Tools. If, for example, you use a - // content delivery network (CDN) to host your images, make sure that the - // hosting site is verified in Webmaster Tools OR that you submit your - // sitemap using robots.txt. In addition, make sure that your robots.txt - // file doesn’t disallow the crawling of any content you want indexed. - URL string - Title string - Caption string - AltText string - - // The geographic location of the image. For example, - // <image:geo_location>Limerick, Ireland</image:geo_location>. - GeoLocation string - - // A URL to the license of the image. - License string -} - -// A Video contains metadata for videos + video sitemaps -// https://support.google.com/webmasters/answer/80471?hl=en -type Video struct { - ThumbnailLoc string - Title string - Description string - ContentLoc string - PlayerLoc string - Duration string - ExpirationDate string - Rating string - ViewCount string - PublicationDate string - FamilyFriendly string - Restriction string - GalleryLoc string - Price string - RequiresSubscription string - Uploader string - Live string -} diff --git a/hugolib/menu.go b/hugolib/menu.go deleted file mode 100644 index 81c136405..000000000 --- a/hugolib/menu.go +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "html/template" - "sort" - "strings" - - "github.com/spf13/cast" -) - -// MenuEntry represents a menu item defined in either Page front matter -// or in the site config. -type MenuEntry struct { - URL string - Page *Page - Name string - Menu string - Identifier string - title string - Pre template.HTML - Post template.HTML - Weight int - Parent string - Children Menu -} - -// Menu is a collection of menu entries. -type Menu []*MenuEntry - -// Menus is a dictionary of menus. -type Menus map[string]*Menu - -// PageMenus is a dictionary of menus defined in the Pages. -type PageMenus map[string]*MenuEntry - -// HasChildren returns whether this menu item has any children. -func (m *MenuEntry) HasChildren() bool { - return m.Children != nil -} - -// KeyName returns the key used to identify this menu entry. -func (m *MenuEntry) KeyName() string { - if m.Identifier != "" { - return m.Identifier - } - return m.Name -} - -func (m *MenuEntry) hopefullyUniqueID() string { - if m.Identifier != "" { - return m.Identifier - } else if m.URL != "" { - return m.URL - } else { - return m.Name - } -} - -// IsEqual returns whether the two menu entries represents the same menu entry. -func (m *MenuEntry) IsEqual(inme *MenuEntry) bool { - return m.hopefullyUniqueID() == inme.hopefullyUniqueID() && m.Parent == inme.Parent -} - -// IsSameResource returns whether the two menu entries points to the same -// resource (URL). -func (m *MenuEntry) IsSameResource(inme *MenuEntry) bool { - return m.URL != "" && inme.URL != "" && m.URL == inme.URL -} - -func (m *MenuEntry) marshallMap(ime map[string]interface{}) { - for k, v := range ime { - loki := strings.ToLower(k) - switch loki { - case "url": - m.URL = cast.ToString(v) - case "weight": - m.Weight = cast.ToInt(v) - case "name": - m.Name = cast.ToString(v) - case "title": - m.title = cast.ToString(v) - case "pre": - m.Pre = template.HTML(cast.ToString(v)) - case "post": - m.Post = template.HTML(cast.ToString(v)) - case "identifier": - m.Identifier = cast.ToString(v) - case "parent": - m.Parent = cast.ToString(v) - } - } -} - -func (m Menu) add(me *MenuEntry) Menu { - app := func(slice Menu, x ...*MenuEntry) Menu { - n := len(slice) + len(x) - if n > cap(slice) { - size := cap(slice) * 2 - if size < n { - size = n - } - new := make(Menu, size) - copy(new, slice) - slice = new - } - slice = slice[0:n] - copy(slice[n-len(x):], x) - return slice - } - - m = app(m, me) - m.Sort() - return m -} - -/* - * Implementation of a custom sorter for Menu - */ - -// A type to implement the sort interface for Menu -type menuSorter struct { - menu Menu - by menuEntryBy -} - -// Closure used in the Sort.Less method. -type menuEntryBy func(m1, m2 *MenuEntry) bool - -func (by menuEntryBy) Sort(menu Menu) { - ms := &menuSorter{ - menu: menu, - by: by, // The Sort method's receiver is the function (closure) that defines the sort order. - } - sort.Stable(ms) -} - -var defaultMenuEntrySort = func(m1, m2 *MenuEntry) bool { - if m1.Weight == m2.Weight { - if m1.Name == m2.Name { - return m1.Identifier < m2.Identifier - } - return m1.Name < m2.Name - } - - if m2.Weight == 0 { - return true - } - - if m1.Weight == 0 { - return false - } - - return m1.Weight < m2.Weight -} - -func (ms *menuSorter) Len() int { return len(ms.menu) } -func (ms *menuSorter) Swap(i, j int) { ms.menu[i], ms.menu[j] = ms.menu[j], ms.menu[i] } - -// Less is part of sort.Interface. It is implemented by calling the "by" closure in the sorter. -func (ms *menuSorter) Less(i, j int) bool { return ms.by(ms.menu[i], ms.menu[j]) } - -// Sort sorts the menu by weight, name and then by identifier. -func (m Menu) Sort() Menu { - menuEntryBy(defaultMenuEntrySort).Sort(m) - return m -} - -// Limit limits the returned menu to n entries. -func (m Menu) Limit(n int) Menu { - if len(m) > n { - return m[0:n] - } - return m -} - -// ByWeight sorts the menu by the weight defined in the menu configuration. -func (m Menu) ByWeight() Menu { - menuEntryBy(defaultMenuEntrySort).Sort(m) - return m -} - -// ByName sorts the menu by the name defined in the menu configuration. -func (m Menu) ByName() Menu { - title := func(m1, m2 *MenuEntry) bool { - return m1.Name < m2.Name - } - - menuEntryBy(title).Sort(m) - return m -} - -// Reverse reverses the order of the menu entries. -func (m Menu) Reverse() Menu { - for i, j := 0, len(m)-1; i < j; i, j = i+1, j-1 { - m[i], m[j] = m[j], m[i] - } - - return m -} - -func (m *MenuEntry) Title() string { - if m.title != "" { - return m.title - } - - if m.Page != nil { - return m.Page.LinkTitle() - } - - return "" -} diff --git a/hugolib/menu_test.go b/hugolib/menu_test.go index 6a8c89b95..253259af1 100644 --- a/hugolib/menu_test.go +++ b/hugolib/menu_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -83,9 +83,9 @@ Menu Main: {{ partial "menu.html" (dict "page" . "menu" "main") }}`, s := h.Sites[0] - require.Len(t, s.Menus, 2) + require.Len(t, s.Menus(), 2) - p1 := s.RegularPages[0].Menus() + p1 := s.RegularPages()[0].Menus() // There is only one menu in the page, but it is "member of" 2 require.Len(t, p1, 1) diff --git a/hugolib/minify_publisher_test.go b/hugolib/minify_publisher_test.go index ce183343b..66e674ade 100644 --- a/hugolib/minify_publisher_test.go +++ b/hugolib/minify_publisher_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,13 +17,10 @@ import ( "testing" "github.com/spf13/viper" - - "github.com/stretchr/testify/require" ) func TestMinifyPublisher(t *testing.T) { t.Parallel() - assert := require.New(t) v := viper.New() v.Set("minify", true) @@ -43,29 +40,24 @@ func TestMinifyPublisher(t *testing.T) { <body id="home"> - <h1>{{ .Page.Title }}</h1> + <h1>{{ .Title }}</h1> + <p>{{ .Permalink }}</p> </body> </html> ` b := newTestSitesBuilder(t) - b.WithViper(v).WithContent("page.md", pageWithAlias) - b.WithTemplates("_default/list.html", htmlTemplate, "_default/single.html", htmlTemplate, "alias.html", htmlTemplate) + b.WithViper(v).WithTemplatesAdded("layouts/index.html", htmlTemplate) b.CreateSites().Build(BuildCfg{}) - assert.Equal(1, len(b.H.Sites)) - require.Len(t, b.H.Sites[0].RegularPages, 1) - // Check minification // HTML - b.AssertFileContent("public/page/index.html", "<!doctype html><html lang=en><head><meta charset=utf-8><title>HTML5 boilerplate – all you really need…</title><link rel=stylesheet href=css/style.css></head><body id=home><h1>Has Alias</h1></body></html>") - // HTML alias. Note the custom template which does no redirect. - b.AssertFileContent("public/foo/bar/index.html", "<!doctype html><html lang=en><head><meta charset=utf-8><title>HTML5 boilerplate ") + b.AssertFileContent("public/index.html", "<!doctype html>") // RSS b.AssertFileContent("public/index.xml", "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?><rss version=\"2.0\" xmlns:atom=\"http://www.w3.org/2005/Atom\"><channel><title/><link>https://example.org/</link>") // Sitemap - b.AssertFileContent("public/sitemap.xml", "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?><urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:xhtml=\"http://www.w3.org/1999/xhtml\"><url><loc>https://example.org/</loc><priority>0</priority></url><url>") + b.AssertFileContent("public/sitemap.xml", "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?><urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:xhtml=\"http://www.w3.org/1999/xhtml\"><url><loc>h") } diff --git a/hugolib/multilingual.go b/hugolib/multilingual.go index c09e3667e..6f744f3a5 100644 --- a/hugolib/multilingual.go +++ b/hugolib/multilingual.go @@ -1,4 +1,4 @@ -// Copyright 2016-present The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -62,10 +62,10 @@ func newMultiLingualFromSites(cfg config.Provider, sites ...*Site) (*Multilingua languages := make(langs.Languages, len(sites)) for i, s := range sites { - if s.Language == nil { - return nil, errors.New("Missing language for site") + if s.language == nil { + return nil, errors.New("missing language for site") } - languages[i] = s.Language + languages[i] = s.language } defaultLang := cfg.GetString("defaultContentLanguage") @@ -78,19 +78,15 @@ func newMultiLingualFromSites(cfg config.Provider, sites ...*Site) (*Multilingua } -func newMultiLingualForLanguage(language *langs.Language) *Multilingual { - languages := langs.Languages{language} - return &Multilingual{Languages: languages, DefaultLang: language} -} func (ml *Multilingual) enabled() bool { return len(ml.Languages) > 1 } func (s *Site) multilingualEnabled() bool { - if s.owner == nil { + if s.h == nil { return false } - return s.owner.multilingual != nil && s.owner.multilingual.enabled() + return s.h.multilingual != nil && s.h.multilingual.enabled() } func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (langs.Languages, error) { diff --git a/hugolib/orderedMap.go b/hugolib/orderedMap.go deleted file mode 100644 index 457cd3d6e..000000000 --- a/hugolib/orderedMap.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "sync" -) - -type orderedMap struct { - sync.RWMutex - keys []interface{} - m map[interface{}]interface{} -} - -func newOrderedMap() *orderedMap { - return &orderedMap{m: make(map[interface{}]interface{})} -} - -func newOrderedMapFromStringMapString(m map[string]string) *orderedMap { - om := newOrderedMap() - for k, v := range m { - om.Add(k, v) - } - return om -} - -func (m *orderedMap) Add(k, v interface{}) { - m.Lock() - defer m.Unlock() - _, found := m.m[k] - if found { - panic(fmt.Sprintf("%v already added", v)) - } - m.m[k] = v - m.keys = append(m.keys, k) -} - -func (m *orderedMap) Get(k interface{}) (interface{}, bool) { - m.RLock() - defer m.RUnlock() - v, found := m.m[k] - return v, found -} - -func (m *orderedMap) Contains(k interface{}) bool { - m.RLock() - defer m.RUnlock() - _, found := m.m[k] - return found -} - -func (m *orderedMap) Keys() []interface{} { - m.RLock() - defer m.RUnlock() - return m.keys -} - -func (m *orderedMap) Len() int { - m.RLock() - defer m.RUnlock() - return len(m.keys) -} - -// Some shortcuts for known types. -func (m *orderedMap) getShortcode(k interface{}) *shortcode { - v, found := m.Get(k) - if !found { - return nil - } - return v.(*shortcode) -} - -func (m *orderedMap) getShortcodeRenderer(k interface{}) func() (string, error) { - v, found := m.Get(k) - if !found { - return nil - } - return v.(func() (string, error)) -} - -func (m *orderedMap) getString(k interface{}) string { - v, found := m.Get(k) - if !found { - return "" - } - return v.(string) -} diff --git a/hugolib/orderedMap_test.go b/hugolib/orderedMap_test.go deleted file mode 100644 index fc3d25080..000000000 --- a/hugolib/orderedMap_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "sync" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestOrderedMap(t *testing.T) { - t.Parallel() - assert := require.New(t) - - m := newOrderedMap() - m.Add("b", "vb") - m.Add("c", "vc") - m.Add("a", "va") - b, f1 := m.Get("b") - - assert.True(f1) - assert.Equal(b, "vb") - assert.True(m.Contains("b")) - assert.False(m.Contains("e")) - - assert.Equal([]interface{}{"b", "c", "a"}, m.Keys()) - -} - -func TestOrderedMapConcurrent(t *testing.T) { - t.Parallel() - assert := require.New(t) - - var wg sync.WaitGroup - - m := newOrderedMap() - - for i := 1; i < 20; i++ { - wg.Add(1) - go func(id int) { - defer wg.Done() - key := fmt.Sprintf("key%d", id) - val := key + "val" - m.Add(key, val) - v, found := m.Get(key) - assert.True(found) - assert.Equal(v, val) - assert.True(m.Contains(key)) - assert.True(m.Len() > 0) - assert.True(len(m.Keys()) > 0) - }(i) - - } - - wg.Wait() -} diff --git a/hugolib/page.go b/hugolib/page.go index 71070d1e8..24d659fb1 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,2012 +15,836 @@ package hugolib import ( "bytes" - "context" - "errors" "fmt" - "math/rand" - "reflect" - - "github.com/gohugoio/hugo/common/hugo" - - "github.com/gohugoio/hugo/common/maps" - "github.com/gohugoio/hugo/common/urls" - "github.com/gohugoio/hugo/media" - - "github.com/gohugoio/hugo/langs" - - "github.com/gohugoio/hugo/related" - - "github.com/bep/gitmap" - - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugolib/pagemeta" - "github.com/gohugoio/hugo/resources/resource" - - "github.com/gohugoio/hugo/output" - "github.com/mitchellh/mapstructure" - "html/template" - "io" + "os" "path" "path/filepath" - "regexp" "runtime" + "sort" "strings" - "sync" - "time" - "unicode/utf8" - "github.com/gohugoio/hugo/compare" - "github.com/gohugoio/hugo/source" + "github.com/bep/gitmap" "github.com/spf13/cast" -) -var ( - cjk = regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}`) + "github.com/gohugoio/hugo/helpers" - // This is all the kinds we can expect to find in .Site.Pages. - allKindsInPages = []string{KindPage, KindHome, KindSection, KindTaxonomy, KindTaxonomyTerm} + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/parser/metadecoders" - allKinds = append(allKindsInPages, []string{kindRSS, kindSitemap, kindRobotsTXT, kind404}...) + "github.com/gohugoio/hugo/parser/pageparser" + "github.com/pkg/errors" - // Assert that it implements the Eqer interface. - _ compare.Eqer = (*Page)(nil) - _ compare.Eqer = (*PageOutput)(nil) + "github.com/gohugoio/hugo/output" - // Assert that it implements the interface needed for related searches. - _ related.Document = (*Page)(nil) + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/source" - // Page supports ref and relref - _ urls.RefLinker = (*Page)(nil) + "github.com/gohugoio/hugo/common/collections" + "github.com/gohugoio/hugo/common/text" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource" ) -// Wraps a Page. -type pageContainer interface { - page() *Page -} - -const ( - KindPage = "page" - - // The rest are node types; home page, sections etc. - - KindHome = "home" - KindSection = "section" - KindTaxonomy = "taxonomy" - KindTaxonomyTerm = "taxonomyTerm" - - // Temporary state. - kindUnknown = "unknown" - - // The following are (currently) temporary nodes, - // i.e. nodes we create just to render in isolation. - kindRSS = "RSS" - kindSitemap = "sitemap" - kindRobotsTXT = "robotsTXT" - kind404 = "404" - - pageResourceType = "page" +var ( + _ page.Page = (*pageState)(nil) + _ collections.Grouper = (*pageState)(nil) + _ collections.Slicer = (*pageState)(nil) ) -type Page struct { - *pageInit - *pageContentInit - - // Kind is the discriminator that identifies the different page types - // in the different page collections. This can, as an example, be used - // to to filter regular pages, find sections etc. - // Kind will, for the pages available to the templates, be one of: - // page, home, section, taxonomy and taxonomyTerm. - // It is of string type to make it easy to reason about in - // the templates. - Kind string - - // Since Hugo 0.18 we got rid of the Node type. So now all pages are ... - // pages (regular pages, home page, sections etc.). - // Sections etc. will have child pages. These were earlier placed in .Data.Pages, - // but can now be more intuitively also be fetched directly from .Pages. - // This collection will be nil for regular pages. - Pages Pages - - // Since Hugo 0.32, a Page can have resources such as images and CSS associated - // with itself. The resource will typically be placed relative to the Page, - // but templates should use the links (Permalink and RelPermalink) - // provided by the Resource object. - Resources resource.Resources - - // This is the raw front matter metadata that is going to be assigned to - // the Resources above. - resourcesMetadata []map[string]interface{} - - // translations will contain references to this page in other language - // if available. - translations Pages - - // A key that maps to translation(s) of this page. This value is fetched - // from the page front matter. - translationKey string - - // Params contains configuration defined in the params section of page frontmatter. - params map[string]interface{} - - // Content sections - contentv template.HTML - summary template.HTML - TableOfContents template.HTML - - // Passed to the shortcodes - pageWithoutContent *PageWithoutContent - - Aliases []string - - Images []Image - Videos []Video - - truncated bool - Draft bool - Status string - - // PageMeta contains page stats such as word count etc. - PageMeta - - // Markup contains the markup type for the content. - Markup string - - extension string - contentType string - - Layout string - - // For npn-renderable pages (see IsRenderable), the content itself - // is used as template and the template name is stored here. - selfLayout string - - linkTitle string - - // Content items. - pageContent - - // whether the content is in a CJK language. - isCJKLanguage bool - - // the content stripped for HTML - plain string // TODO should be []byte - plainWords []string - - // rendering configuration - renderingConfig *helpers.BlackFriday - - // menus - pageMenus PageMenus - - source.File - - Position `json:"-"` - - GitInfo *gitmap.GitInfo - - // This was added as part of getting the Nodes (taxonomies etc.) to work as - // Pages in Hugo 0.18. - // It is deliberately named similar to Section, but not exported (for now). - // We currently have only one level of section in Hugo, but the page can live - // any number of levels down the file path. - // To support taxonomies like /categories/hugo etc. we will need to keep track - // of that information in a general way. - // So, sections represents the path to the content, i.e. a content file or a - // virtual content file in the situations where a taxonomy or a section etc. - // isn't accomanied by one. - sections []string - - // Will only be set for sections and regular pages. - parent *Page - - // When we create paginator pages, we create a copy of the original, - // but keep track of it here. - origOnCopy *Page - - // Will only be set for section pages and the home page. - subSections Pages - - s *Site - - // Pulled over from old Node. TODO(bep) reorg and group (embed) - - Site *SiteInfo `json:"-"` - - title string - Description string - Keywords []string - data map[string]interface{} - - pagemeta.PageDates - - Sitemap Sitemap - pagemeta.URLPath - frontMatterURL string - - permalink string - relPermalink string - - // relative target path without extension and any base path element - // from the baseURL or the language code. - // This is used to construct paths in the page resources. - relTargetPathBase string - // Is set to a forward slashed path if this is a Page resources living in a folder below its owner. - resourcePath string - - // This is enabled if it is a leaf bundle (the "index.md" type) and it is marked as headless in front matter. - // Being headless means that - // 1. The page itself is not rendered to disk - // 2. It is not available in .Site.Pages etc. - // 3. But you can get it via .Site.GetPage - headless bool - - layoutDescriptor output.LayoutDescriptor - - scratch *maps.Scratch - - // It would be tempting to use the language set on the Site, but in they way we do - // multi-site processing, these values may differ during the initial page processing. - language *langs.Language - - lang string - - // When in Fast Render Mode, we only render a sub set of the pages, i.e. the - // pages the user is working on. There are, however, situations where we need to - // signal other pages to be rendered. - forceRender bool - - // The output formats this page will be rendered to. - outputFormats output.Formats - - // This is the PageOutput that represents the first item in outputFormats. - // Use with care, as there are potential for inifinite loops. - mainPageOutput *PageOutput - - targetPathDescriptorPrototype *targetPathDescriptor -} - -func stackTrace(length int) string { - trace := make([]byte, length) - runtime.Stack(trace, true) - return string(trace) -} - -func (p *Page) Data() interface{} { - return p.data -} - -func (p *Page) initContent() { - - p.contentInit.Do(func() { - // This careful dance is here to protect against circular loops in shortcode/content - // constructs. - // TODO(bep) context vs the remote shortcodes - ctx, cancel := context.WithTimeout(context.Background(), p.s.Timeout) - defer cancel() - c := make(chan error, 1) - - p.contentInitMu.Lock() - defer p.contentInitMu.Unlock() - - go func() { - var err error - - err = p.prepareContent() - if err != nil { - c <- err - return - } - - select { - case <-ctx.Done(): - return - default: - } - - if len(p.summary) == 0 { - if err = p.setAutoSummary(); err != nil { - err = p.errorf(err, "failed to set auto summary") - } - } - c <- err - }() - - select { - case <-ctx.Done(): - p.s.Log.WARN.Printf("Timed out creating content for page %q (.Content will be empty). This is most likely a circular shortcode content loop that should be fixed. If this is just a shortcode calling a slow remote service, try to set \"timeout=30000\" (or higher, value is in milliseconds) in config.toml.\n", p.pathOrTitle()) - case err := <-c: - if err != nil { - p.s.SendError(err) - } - } - }) - -} - -// This is sent to the shortcodes for this page. Not doing that will create an infinite regress. So, -// shortcodes can access .Page.TableOfContents, but not .Page.Content etc. -func (p *Page) withoutContent() *PageWithoutContent { - p.pageInit.withoutContentInit.Do(func() { - p.pageWithoutContent = &PageWithoutContent{Page: p} - }) - return p.pageWithoutContent -} - -func (p *Page) Content() (interface{}, error) { - return p.content(), nil -} - -func (p *Page) Truncated() bool { - p.initContent() - return p.truncated -} - -func (p *Page) content() template.HTML { - p.initContent() - return p.contentv -} - -func (p *Page) Summary() template.HTML { - p.initContent() - return p.summary -} - -// Sites is a convenience method to get all the Hugo sites/languages configured. -func (p *Page) Sites() SiteInfos { - return p.s.owner.siteInfos() -} - -// SearchKeywords implements the related.Document interface needed for fast page searches. -func (p *Page) SearchKeywords(cfg related.IndexConfig) ([]related.Keyword, error) { - - v, err := p.Param(cfg.Name) - if err != nil { - return nil, err - } +var ( + pageTypesProvider = resource.NewResourceTypesProvider(media.OctetType, pageResourceType) + nopPageOutput = &pageOutput{pagePerOutputProviders: nopPagePerOutput} +) - return cfg.ToKeywords(v) +// pageContext provides contextual information about this page, for error +// logging and similar. +type pageContext interface { + posOffset(offset int) text.Position + wrapError(err error) error + getRenderingConfig() *helpers.BlackFriday } -// PubDate is when this page was or will be published. -// NOTE: This is currently used for search only and is not meant to be used -// directly in templates. We need to consolidate the dates in this struct. -// TODO(bep) see https://github.com/gohugoio/hugo/issues/3854 -func (p *Page) PubDate() time.Time { - if !p.PublishDate.IsZero() { - return p.PublishDate +// wrapErr adds some context to the given error if possible. +func wrapErr(err error, ctx interface{}) error { + if pc, ok := ctx.(pageContext); ok { + return pc.wrapError(err) } - return p.Date + return err } -func (*Page) ResourceType() string { - return pageResourceType -} - -func (p *Page) RSSLink() template.URL { - f, found := p.outputFormats.GetByName(output.RSSFormat.Name) - if !found { - return "" - } - return template.URL(newOutputFormat(p, f).Permalink()) +type pageSiteAdapter struct { + p page.Page + s *Site } -func (p *Page) createLayoutDescriptor() output.LayoutDescriptor { - var section string - - switch p.Kind { - case KindSection: - // In Hugo 0.22 we introduce nested sections, but we still only - // use the first level to pick the correct template. This may change in - // the future. - section = p.sections[0] - case KindTaxonomy, KindTaxonomyTerm: - section = p.s.taxonomiesPluralSingular[p.sections[0]] - default: - } - - return output.LayoutDescriptor{ - Kind: p.Kind, - Type: p.Type(), - Lang: p.Lang(), - Layout: p.Layout, - Section: section, +func (pa pageSiteAdapter) GetPage(ref string) (page.Page, error) { + p, err := pa.s.getPageNew(pa.p, ref) + if p == nil { + // The nil struct has meaning in some situations, mostly to avoid breaking + // existing sites doing $nilpage.IsDescendant($p), which will always return + // false. + p = page.NilPage } + return p, err } -// pageInit lazy initializes different parts of the page. It is extracted -// into its own type so we can easily create a copy of a given page. -type pageInit struct { - languageInit sync.Once - pageMenusInit sync.Once - pageMetaInit sync.Once - renderingConfigInit sync.Once - withoutContentInit sync.Once -} +type pageState struct { + // This slice will be of same length as the number of global slice of output + // formats (for all sites). + pageOutputs []*pageOutput -type pageContentInit struct { - contentInitMu sync.Mutex - contentInit sync.Once - plainInit sync.Once - plainWordsInit sync.Once -} + // This will be shifted out when we start to render a new output format. + *pageOutput -func (p *Page) resetContent() { - p.pageContentInit = &pageContentInit{} + // Common for all output formats. + *pageCommon } -// IsNode returns whether this is an item of one of the list types in Hugo, -// i.e. not a regular content page. -func (p *Page) IsNode() bool { - return p.Kind != KindPage -} +// Eq returns whether the current page equals the given page. +// This is what's invoked when doing `{{ if eq $page $otherPage }}` +func (p *pageState) Eq(other interface{}) bool { + pp, err := unwrapPage(other) + if err != nil { + return false + } -// IsHome returns whether this is the home page. -func (p *Page) IsHome() bool { - return p.Kind == KindHome + return p == pp } -// IsSection returns whether this is a section page. -func (p *Page) IsSection() bool { - return p.Kind == KindSection +func (p *pageState) GitInfo() *gitmap.GitInfo { + return p.gitInfo } -// IsPage returns whether this is a regular content page. -func (p *Page) IsPage() bool { - return p.Kind == KindPage +func (p *pageState) MarshalJSON() ([]byte, error) { + return page.MarshalPageToJSON(p) } -// BundleType returns the bundle type: "leaf", "branch" or an empty string if it is none. -// See https://gohugo.io/content-management/page-bundles/ -func (p *Page) BundleType() string { - if p.IsNode() { - return "branch" - } - - var source interface{} = p.File - if fi, ok := source.(*fileInfo); ok { - switch fi.bundleTp { - case bundleBranch: - return "branch" - case bundleLeaf: - return "leaf" +func (p *pageState) Pages() page.Pages { + p.pagesInit.Do(func() { + if p.pages != nil { + return } - } - - return "" -} -func (p *Page) MediaType() media.Type { - return media.OctetType -} + var pages page.Pages -type PageMeta struct { - wordCount int - fuzzyWordCount int - readingTime int - Weight int -} - -type Position struct { - PrevPage *Page - NextPage *Page - PrevInSection *Page - NextInSection *Page -} + switch p.Kind() { + case page.KindPage: + case page.KindHome: + pages = p.s.RegularPages() + case page.KindTaxonomy: + termInfo := p.getTaxonomyNodeInfo() + taxonomy := p.s.Taxonomies[termInfo.plural].Get(termInfo.termKey) + pages = taxonomy.Pages() + case page.KindTaxonomyTerm: + plural := p.getTaxonomyNodeInfo().plural + // A list of all page.KindTaxonomy pages with matching plural + for _, p := range p.s.findPagesByKind(page.KindTaxonomy) { + if p.SectionsEntries()[0] == plural { + pages = append(pages, p) + } + } + case kind404, kindSitemap, kindRobotsTXT: + pages = p.s.Pages() + } -type Pages []*Page + p.pages = pages + }) -func (ps Pages) String() string { - return fmt.Sprintf("Pages(%d)", len(ps)) + return p.pages } -// Used in tests. -func (ps Pages) shuffle() { - for i := range ps { - j := rand.Intn(i + 1) - ps[i], ps[j] = ps[j], ps[i] +// RawContent returns the un-rendered source content without +// any leading front matter. +func (p *pageState) RawContent() string { + if p.source.parsed == nil { + return "" } -} - -func (ps Pages) findPagePosByFilename(filename string) int { - for i, x := range ps { - if x.Filename() == filename { - return i - } + start := p.source.posMainContent + if start == -1 { + start = 0 } - return -1 + return string(p.source.parsed.Input()[start:]) } -func (ps Pages) removeFirstIfFound(p *Page) Pages { - ii := -1 - for i, pp := range ps { - if pp == p { - ii = i - break - } - } +func (p *pageState) Resources() resource.Resources { + p.resourcesInit.Do(func() { - if ii != -1 { - ps = append(ps[:ii], ps[ii+1:]...) - } - return ps -} + sort := func() { + sort.SliceStable(p.resources, func(i, j int) bool { + ri, rj := p.resources[i], p.resources[j] + if ri.ResourceType() < rj.ResourceType() { + return true + } -func (ps Pages) findPagePosByFilnamePrefix(prefix string) int { - if prefix == "" { - return -1 - } + p1, ok1 := ri.(page.Page) + p2, ok2 := rj.(page.Page) - lenDiff := -1 - currPos := -1 - prefixLen := len(prefix) + if ok1 != ok2 { + return ok2 + } - // Find the closest match - for i, x := range ps { - if strings.HasPrefix(x.Filename(), prefix) { - diff := len(x.Filename()) - prefixLen - if lenDiff == -1 || diff < lenDiff { - lenDiff = diff - currPos = i - } - } - } - return currPos -} + if ok1 { + return page.DefaultPageSort(p1, p2) + } -// findPagePos Given a page, it will find the position in Pages -// will return -1 if not found -func (ps Pages) findPagePos(page *Page) int { - for i, x := range ps { - if x.Filename() == page.Filename() { - return i + return ri.RelPermalink() < rj.RelPermalink() + }) } - } - return -1 -} -func (p *Page) Plain() string { - p.initContent() - p.initPlain(true) - return p.plain -} + sort() -func (p *Page) initPlain(lock bool) { - p.plainInit.Do(func() { - if lock { - p.contentInitMu.Lock() - defer p.contentInitMu.Unlock() + if len(p.m.resourcesMetadata) > 0 { + resources.AssignMetadata(p.m.resourcesMetadata, p.resources...) + sort() } - p.plain = helpers.StripHTML(string(p.contentv)) - }) -} - -func (p *Page) PlainWords() []string { - p.initContent() - p.initPlainWords(true) - return p.plainWords -} -func (p *Page) initPlainWords(lock bool) { - p.plainWordsInit.Do(func() { - if lock { - p.contentInitMu.Lock() - defer p.contentInitMu.Unlock() - } - p.plainWords = strings.Fields(p.plain) }) + return p.resources } -// Param is a convenience method to do lookups in Page's and Site's Params map, -// in that order. -// -// This method is also implemented on Node and SiteInfo. -func (p *Page) Param(key interface{}) (interface{}, error) { - keyStr, err := cast.ToStringE(key) - if err != nil { - return nil, err - } - - keyStr = strings.ToLower(keyStr) - result, _ := p.traverseDirect(keyStr) - if result != nil { - return result, nil - } - - keySegments := strings.Split(keyStr, ".") - if len(keySegments) == 1 { - return nil, nil - } - - return p.traverseNested(keySegments) -} - -func (p *Page) traverseDirect(key string) (interface{}, error) { - keyStr := strings.ToLower(key) - if val, ok := p.params[keyStr]; ok { - return val, nil +func (p *pageState) HasShortcode(name string) bool { + if p.shortcodeState == nil { + return false } - return p.Site.Params[keyStr], nil + return p.shortcodeState.nameSet[name] } -func (p *Page) traverseNested(keySegments []string) (interface{}, error) { - result := traverse(keySegments, p.params) - if result != nil { - return result, nil - } - - result = traverse(keySegments, p.Site.Params) - if result != nil { - return result, nil - } - - // Didn't find anything, but also no problems. - return nil, nil +func (p *pageState) Site() page.Site { + return &p.s.Info } -func traverse(keys []string, m map[string]interface{}) interface{} { - // Shift first element off. - firstKey, rest := keys[0], keys[1:] - result := m[firstKey] - - // No point in continuing here. - if result == nil { - return result - } - - if len(rest) == 0 { - // That was the last key. - return result +func (p *pageState) String() string { + if sourceRef := p.sourceRef(); sourceRef != "" { + return fmt.Sprintf("Page(%s)", sourceRef) } - - // That was not the last key. - return traverse(rest, cast.ToStringMap(result)) + return fmt.Sprintf("Page(%q)", p.Title()) } -func (p *Page) Author() Author { - authors := p.Authors() - - for _, author := range authors { - return author - } - return Author{} +// IsTranslated returns whether this content file is translated to +// other language(s). +func (p *pageState) IsTranslated() bool { + p.s.h.init.translations.Do() + return len(p.translations) > 0 } -func (p *Page) Authors() AuthorList { - authorKeys, ok := p.params["authors"] - if !ok { - return AuthorList{} - } - authors := authorKeys.([]string) - if len(authors) < 1 || len(p.Site.Authors) < 1 { - return AuthorList{} - } - - al := make(AuthorList) - for _, author := range authors { - a, ok := p.Site.Authors[author] - if ok { - al[author] = a +// TranslationKey returns the key used to map language translations of this page. +// It will use the translationKey set in front matter if set, or the content path and +// filename (excluding any language code and extension), e.g. "about/index". +// The Page Kind is always prepended. +func (p *pageState) TranslationKey() string { + p.translationKeyInit.Do(func() { + if p.m.translationKey != "" { + p.translationKey = p.Kind() + "/" + p.m.translationKey + } else if p.IsPage() && p.File() != nil { + p.translationKey = path.Join(p.Kind(), filepath.ToSlash(p.File().Dir()), p.File().TranslationBaseName()) + } else if p.IsNode() { + p.translationKey = path.Join(p.Kind(), p.SectionsPath()) } - } - return al -} -func (p *Page) UniqueID() string { - return p.File.UniqueID() -} - -// Returns the page as summary and main. -func (p *Page) setUserDefinedSummary(rawContentCopy []byte) (*summaryContent, error) { - - sc, err := splitUserDefinedSummaryAndContent(p.Markup, rawContentCopy) - - if err != nil { - return nil, err - } - - if sc == nil { - // No divider found - return nil, nil - } + }) - p.summary = helpers.BytesToHTML(sc.summary) + return p.translationKey - return sc, nil } -// Make this explicit so there is no doubt about what is what. -type summaryContent struct { - summary []byte - content []byte +// AllTranslations returns all translations, including the current Page. +func (p *pageState) AllTranslations() page.Pages { + p.s.h.init.translations.Do() + return p.allTranslations } -func splitUserDefinedSummaryAndContent(markup string, c []byte) (sc *summaryContent, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("summary split failed: %s", r) - } - }() - - startDivider := bytes.Index(c, internalSummaryDividerBaseBytes) - - if startDivider == -1 { - return - } - - startTag := "p" - switch markup { - case "asciidoc": - startTag = "div" - - } - - // Walk back and forward to the surrounding tags. - start := bytes.LastIndex(c[:startDivider], []byte("<"+startTag)) - end := bytes.Index(c[startDivider:], []byte("</"+startTag)) - - if start == -1 { - start = startDivider - } else { - start = startDivider - (startDivider - start) - } - - if end == -1 { - end = startDivider + len(internalSummaryDividerBase) - } else { - end = startDivider + end + len(startTag) + 3 - } - - var addDiv bool - - switch markup { - case "rst": - addDiv = true - } - - withoutDivider := append(c[:start], bytes.Trim(c[end:], "\n")...) - - var summary []byte - - if len(withoutDivider) > 0 { - summary = bytes.TrimSpace(withoutDivider[:start]) - } - - if addDiv { - // For the rst - summary = append(append([]byte(nil), summary...), []byte("</div>")...) - } - - if err != nil { - return - } +// Translations returns the translations excluding the current Page. +func (p *pageState) Translations() page.Pages { + p.s.h.init.translations.Do() + return p.translations +} - sc = &summaryContent{ - summary: summary, - content: bytes.TrimSpace(withoutDivider), +func (p *pageState) getRenderingConfig() *helpers.BlackFriday { + if p.m.renderingConfig == nil { + return p.s.ContentSpec.BlackFriday } - - return + return p.m.renderingConfig } -func (p *Page) setAutoSummary() error { - var summary string - var truncated bool - // This careful init dance could probably be refined, but it is purely for performance - // reasons. These "plain" methods are expensive if the plain content is never actually - // used. - p.initPlain(false) - if p.isCJKLanguage { - p.initPlainWords(false) - summary, truncated = p.s.ContentSpec.TruncateWordsByRune(p.plainWords) - } else { - summary, truncated = p.s.ContentSpec.TruncateWordsToWholeSentence(p.plain) +func (ps *pageState) initCommonProviders(pp pagePaths) error { + if ps.IsPage() { + ps.posNextPrev = &nextPrev{init: ps.s.init.prevNext} + ps.posNextPrevSection = &nextPrev{init: ps.s.init.prevNextInSection} + ps.InSectionPositioner = newPagePositionInSection(ps.posNextPrevSection) + ps.Positioner = newPagePosition(ps.posNextPrev) } - p.summary = template.HTML(summary) - p.truncated = truncated - return nil - -} + ps.OutputFormatsProvider = pp + ps.targetPathDescriptor = pp.targetPathDescriptor + ps.RefProvider = newPageRef(ps) + ps.SitesProvider = &ps.s.Info -func (p *Page) renderContent(content []byte) []byte { - return p.s.ContentSpec.RenderBytes(&helpers.RenderingContext{ - Content: content, RenderTOC: true, PageFmt: p.Markup, - Cfg: p.Language(), - DocumentID: p.UniqueID(), DocumentName: p.Path(), - Config: p.getRenderingConfig()}) + return nil } -func (p *Page) getRenderingConfig() *helpers.BlackFriday { - p.renderingConfigInit.Do(func() { - bfParam := p.getParamToLower("blackfriday") - if bfParam == nil { - p.renderingConfig = p.s.ContentSpec.BlackFriday - return - } - // Create a copy so we can modify it. - bf := *p.s.ContentSpec.BlackFriday - p.renderingConfig = &bf +func (p *pageState) getLayoutDescriptor() output.LayoutDescriptor { + p.layoutDescriptorInit.Do(func() { + var section string + sections := p.SectionsEntries() - if p.Language() == nil { - panic(fmt.Sprintf("nil language for %s with source lang %s", p.BaseFileName(), p.lang)) + switch p.Kind() { + case page.KindSection: + section = sections[0] + case page.KindTaxonomyTerm: + section = p.getTaxonomyNodeInfo().singular + case page.KindTaxonomy: + section = p.getTaxonomyNodeInfo().parent.singular + default: } - pageParam := cast.ToStringMap(bfParam) - if err := mapstructure.Decode(pageParam, &p.renderingConfig); err != nil { - p.s.Log.FATAL.Printf("Failed to get rendering config for %s:\n%s", p.BaseFileName(), err.Error()) + p.layoutDescriptor = output.LayoutDescriptor{ + Kind: p.Kind(), + Type: p.Type(), + Lang: p.Language().Lang, + Layout: p.Layout(), + Section: section, } - }) - return p.renderingConfig -} + return p.layoutDescriptor -func (s *Site) newPage(filename string) *Page { - fi := newFileInfo( - s.SourceSpec, - s.absContentDir(), - filename, - nil, - bundleNot, - ) - return s.newPageFromFile(fi) } -func (s *Site) newPageFromFile(fi *fileInfo) *Page { - return &Page{ - pageInit: &pageInit{}, - pageContentInit: &pageContentInit{}, - Kind: kindFromFileInfo(fi), - contentType: "", - File: fi, - Keywords: []string{}, Sitemap: Sitemap{Priority: -1}, - params: make(map[string]interface{}), - translations: make(Pages, 0), - sections: sectionsFromFile(fi), - Site: &s.Info, - s: s, - } -} - -func (p *Page) IsRenderable() bool { - return p.renderable -} +func (p *pageState) getLayouts(layouts ...string) ([]string, error) { + f := p.outputFormat() -func (p *Page) Type() string { - if p.contentType != "" { - return p.contentType - } - - if x := p.Section(); x != "" { - return x - } - - return "page" -} - -// Section returns the first path element below the content root. Note that -// since Hugo 0.22 we support nested sections, but this will always be the first -// element of any nested path. -func (p *Page) Section() string { - if p.Kind == KindSection || p.Kind == KindTaxonomy || p.Kind == KindTaxonomyTerm { - return p.sections[0] - } - return p.File.Section() -} - -func (s *Site) newPageFrom(buf io.Reader, name string) (*Page, error) { - p, err := s.NewPage(name) - if err != nil { - return p, err - } - _, err = p.ReadFrom(buf) - if err != nil { - return nil, err + if len(layouts) == 0 { + selfLayout := p.selfLayoutForOutput(f) + if selfLayout != "" { + return []string{selfLayout}, nil + } } - return p, err -} + layoutDescriptor := p.getLayoutDescriptor() -func (s *Site) NewPage(name string) (*Page, error) { - if len(name) == 0 { - return nil, errors.New("Zero length page name") + if len(layouts) > 0 { + layoutDescriptor.Layout = layouts[0] + layoutDescriptor.LayoutOverride = true } - // Create new page - p := s.newPage(name) - p.s = s - p.Site = &s.Info - - return p, nil + return p.s.layoutHandler.For(layoutDescriptor, f) } -func (p *Page) ReadFrom(buf io.Reader) (int64, error) { - // Parse for metadata & body - if err := p.parse(buf); err != nil { - return 0, p.errWithFileContext(err) - +// This is serialized +func (p *pageState) initOutputFormat(idx int) error { + if err := p.shiftToOutputFormat(idx); err != nil { + return err } - if err := p.mapContent(); err != nil { - return 0, p.errWithFileContext(err) + if !p.renderable { + if _, err := p.Content(); err != nil { + return err + } } - return int64(len(p.source.parsed.Input())), nil -} - -func (p *Page) WordCount() int { - p.initContentPlainAndMeta() - return p.wordCount -} + return nil -func (p *Page) ReadingTime() int { - p.initContentPlainAndMeta() - return p.readingTime } -func (p *Page) FuzzyWordCount() int { - p.initContentPlainAndMeta() - return p.fuzzyWordCount +// Must be run after the site section tree etc. is built and ready. +func (p *pageState) initPage() error { + if _, err := p.init.Do(); err != nil { + return err + } + return nil } -func (p *Page) initContentPlainAndMeta() { - p.initContent() - p.initPlain(true) - p.initPlainWords(true) - p.initMeta() +func (p *pageState) setPages(pages page.Pages) { + page.SortByDefault(pages) + p.pages = pages } -func (p *Page) initContentAndMeta() { - p.initContent() - p.initMeta() -} +func (p *pageState) renderResources() error { + var toBeDeleted []int -func (p *Page) initMeta() { - p.pageMetaInit.Do(func() { - if p.isCJKLanguage { - p.wordCount = 0 - for _, word := range p.plainWords { - runeCount := utf8.RuneCountInString(word) - if len(word) == runeCount { - p.wordCount++ - } else { - p.wordCount += runeCount - } - } - } else { - p.wordCount = helpers.TotalWords(p.plain) + for i, r := range p.Resources() { + if _, ok := r.(page.Page); ok { + // Pages gets rendered with the owning page but we count them here. + p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Pages) + continue } - // TODO(bep) is set in a test. Fix that. - if p.fuzzyWordCount == 0 { - p.fuzzyWordCount = (p.wordCount + 100) / 100 * 100 + src, ok := r.(resource.Source) + if !ok { + return errors.Errorf("Resource %T does not support resource.Source", src) } - if p.isCJKLanguage { - p.readingTime = (p.wordCount + 500) / 501 + if err := src.Publish(); err != nil { + if os.IsNotExist(err) { + // The resource has been deleted from the file system. + // This should be extremely rare, but can happen on live reload in server + // mode when the same resource is member of different page bundles. + toBeDeleted = append(toBeDeleted, i) + } else { + p.s.Log.ERROR.Printf("Failed to publish Resource for page %q: %s", p.pathOrTitle(), err) + } } else { - p.readingTime = (p.wordCount + 212) / 213 + p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Files) } - }) -} + } -// HasShortcode return whether the page has a shortcode with the given name. -// This method is mainly motivated with the Hugo Docs site's need for a list -// of pages with the `todo` shortcode in it. -func (p *Page) HasShortcode(name string) bool { - if p.shortcodeState == nil { - return false + for _, i := range toBeDeleted { + p.deleteResource(i) } - return p.shortcodeState.nameSet[name] + return nil } -// AllTranslations returns all translations, including the current Page. -func (p *Page) AllTranslations() Pages { - return p.translations +func (p *pageState) deleteResource(i int) { + p.resources = append(p.resources[:i], p.resources[i+1:]...) } -// IsTranslated returns whether this content file is translated to -// other language(s). -func (p *Page) IsTranslated() bool { - return len(p.translations) > 1 +func (p *pageState) getTargetPaths() page.TargetPaths { + return p.targetPaths() } -// Translations returns the translations excluding the current Page. -func (p *Page) Translations() Pages { - translations := make(Pages, 0) - for _, t := range p.translations { - if t.Lang() != p.Lang() { +func (p *pageState) setTranslations(pages page.Pages) { + p.allTranslations = pages + page.SortByLanguage(p.allTranslations) + translations := make(page.Pages, 0) + for _, t := range p.allTranslations { + if !t.Eq(p) { translations = append(translations, t) } } - return translations -} - -// TranslationKey returns the key used to map language translations of this page. -// It will use the translationKey set in front matter if set, or the content path and -// filename (excluding any language code and extension), e.g. "about/index". -// The Page Kind is always prepended. -func (p *Page) TranslationKey() string { - if p.translationKey != "" { - return p.Kind + "/" + p.translationKey - } - - if p.IsNode() { - return path.Join(p.Kind, path.Join(p.sections...), p.TranslationBaseName()) - } - - return path.Join(p.Kind, filepath.ToSlash(p.Dir()), p.TranslationBaseName()) -} - -func (p *Page) LinkTitle() string { - if len(p.linkTitle) > 0 { - return p.linkTitle - } - return p.title + p.translations = translations } -func (p *Page) shouldBuild() bool { - return shouldBuild(p.s.BuildFuture, p.s.BuildExpired, - p.s.BuildDrafts, p.Draft, p.PublishDate, p.ExpiryDate) -} +func (p *pageState) AlternativeOutputFormats() page.OutputFormats { + f := p.outputFormat() + var o page.OutputFormats + for _, of := range p.OutputFormats() { + if of.Format.NotAlternative || of.Format.Name == f.Name { + continue + } -func shouldBuild(buildFuture bool, buildExpired bool, buildDrafts bool, Draft bool, - publishDate time.Time, expiryDate time.Time) bool { - if !(buildDrafts || !Draft) { - return false - } - if !buildFuture && !publishDate.IsZero() && publishDate.After(time.Now()) { - return false - } - if !buildExpired && !expiryDate.IsZero() && expiryDate.Before(time.Now()) { - return false + o = append(o, of) } - return true + return o } -func (p *Page) IsDraft() bool { - return p.Draft -} - -func (p *Page) IsFuture() bool { - if p.PublishDate.IsZero() { - return false +func (p *pageState) Render(layout ...string) template.HTML { + l, err := p.getLayouts(layout...) + if err != nil { + p.s.SendError(p.wrapError(errors.Errorf(".Render: failed to resolve layout %v", layout))) + return "" } - return p.PublishDate.After(time.Now()) -} -func (p *Page) IsExpired() bool { - if p.ExpiryDate.IsZero() { - return false + for _, layout := range l { + templ, found := p.s.Tmpl.Lookup(layout) + if !found { + // This is legacy from when we had only one output format and + // HTML templates only. Some have references to layouts without suffix. + // We default to good old HTML. + templ, _ = p.s.Tmpl.Lookup(layout + ".html") + } + if templ != nil { + res, err := executeToString(templ, p) + if err != nil { + p.s.SendError(p.wrapError(errors.Wrapf(err, ".Render: failed to execute template %q v", layout))) + return "" + } + return template.HTML(res) + } } - return p.ExpiryDate.Before(time.Now()) -} -func (p *Page) URL() string { + return "" - if p.IsPage() && p.URLPath.URL != "" { - // This is the url set in front matter - return p.URLPath.URL - } - // Fall back to the relative permalink. - u := p.RelPermalink() - return u } -// Permalink returns the absolute URL to this Page. -func (p *Page) Permalink() string { - if p.headless { - return "" - } - return p.permalink -} +// wrapError adds some more context to the given error if possible +func (p *pageState) wrapError(err error) error { -// RelPermalink gets a URL to the resource relative to the host. -func (p *Page) RelPermalink() string { - if p.headless { - return "" + var filename string + if p.File() != nil { + filename = p.File().Filename() } - return p.relPermalink -} -// See resource.Resource -// This value is used, by default, in Resources.ByPrefix etc. -func (p *Page) Name() string { - if p.resourcePath != "" { - return p.resourcePath - } - return p.title -} - -func (p *Page) Title() string { - return p.title -} + err, _ = herrors.WithFileContextForFile( + err, + filename, + filename, + p.s.SourceSpec.Fs.Source, + herrors.SimpleLineMatcher) -func (p *Page) Params() map[string]interface{} { - return p.params + return err } -func (p *Page) subResourceTargetPathFactory(base string) string { - return path.Join(p.relTargetPathBase, base) +func (p *pageState) addResources(r ...resource.Resource) { + p.resources = append(p.resources, r...) } -// Prepare this page for rendering for a new site. The flag start is set -// for the first site and output format. -func (p *Page) prepareForRender(start bool) error { - p.setContentInit(start) - if start { - return p.initMainOutputFormat() +func (p *pageState) addSectionToParent() { + if p.parent == nil { + return } - return nil + p.parent.subSections = append(p.parent.subSections, p) } -func (p *Page) initMainOutputFormat() error { - outFormat := p.outputFormats[0] - pageOutput, err := newPageOutput(p, false, false, outFormat) +func (p *pageState) contentMarkupType() string { + if p.m.markup != "" { + return p.m.markup - if err != nil { - return p.errorf(err, "failed to create output page for type %q", outFormat.Name) } - - p.mainPageOutput = pageOutput - - return nil - + return p.File().Ext() } -func (p *Page) setContentInit(start bool) error { +func (p *pageState) mapContent(meta *pageMeta) error { - if start { - // This is a new language. - p.shortcodeState.clearDelta() - } - updated := true - if p.shortcodeState != nil { - updated = p.shortcodeState.updateDelta() - } + s := p.shortcodeState - if updated { - p.resetContent() - } + p.renderable = true - for _, r := range p.Resources.ByType(pageResourceType) { - p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Pages) - bp := r.(*Page) - if start { - bp.shortcodeState.clearDelta() - } - if bp.shortcodeState != nil { - updated = bp.shortcodeState.updateDelta() - } - if updated { - bp.resetContent() - } + rn := &pageContentMap{ + items: make([]interface{}, 0, 20), } - return nil - -} - -func (p *Page) prepareContent() error { - s := p.s - - // If we got this far it means that this is either a new Page pointer - // or a template or similar has changed so wee need to do a rerendering - // of the shortcodes etc. - - // If in watch mode or if we have multiple sites or output formats, - // we need to keep the original so we can - // potentially repeat this process on rebuild. - needsACopy := s.running() || len(s.owner.Sites) > 1 || len(p.outputFormats) > 1 - var workContentCopy []byte - if needsACopy { - workContentCopy = make([]byte, len(p.workContent)) - copy(workContentCopy, p.workContent) - } else { - // Just reuse the same slice. - workContentCopy = p.workContent - } + iter := p.source.parsed.Iterator() - var err error - // Note: The shortcodes in a page cannot access the page content it lives in, - // hence the withoutContent(). - if workContentCopy, err = handleShortcodes(p.withoutContent(), workContentCopy); err != nil { - return err + fail := func(err error, i pageparser.Item) error { + return p.parseError(err, iter.Input(), i.Pos) } - if p.Markup != "html" && p.source.hasSummaryDivider { - - // Now we know enough to create a summary of the page and count some words - summaryContent, err := p.setUserDefinedSummary(workContentCopy) + // the parser is guaranteed to return items in proper order or fail, so … + // … it's safe to keep some "global" state + var currShortcode shortcode + var ordinal int - if err != nil { - s.Log.ERROR.Printf("Failed to set user defined summary for page %q: %s", p.Path(), err) - } else if summaryContent != nil { - workContentCopy = summaryContent.content - } +Loop: + for { + it := iter.Next() - p.contentv = helpers.BytesToHTML(workContentCopy) - - } else { - p.contentv = helpers.BytesToHTML(workContentCopy) - } + switch { + case it.Type == pageparser.TypeIgnore: + case it.Type == pageparser.TypeHTMLStart: + // This is HTML without front matter. It can still have shortcodes. + p.selfLayout = "__" + p.File().Filename() + p.renderable = false + rn.AddBytes(it) + case it.IsFrontMatter(): + f := metadecoders.FormatFromFrontMatterType(it.Type) + m, err := metadecoders.Default.UnmarshalToMap(it.Val, f) + if err != nil { + if fe, ok := err.(herrors.FileError); ok { + return herrors.ToFileErrorWithOffset(fe, iter.LineNumber()-1) + } else { + return err + } + } - return nil -} + if err := meta.setMetadata(p, m); err != nil { + return err + } -func (p *Page) updateMetaData(frontmatter map[string]interface{}) error { - if frontmatter == nil { - return errors.New("missing frontmatter data") - } - // Needed for case insensitive fetching of params values - maps.ToLower(frontmatter) + next := iter.Peek() + if !next.IsDone() { + p.source.posMainContent = next.Pos + } - var mtime time.Time - if p.FileInfo() != nil { - mtime = p.FileInfo().ModTime() - } + if !p.s.shouldBuild(p) { + // Nothing more to do. + return nil + } - var gitAuthorDate time.Time - if p.GitInfo != nil { - gitAuthorDate = p.GitInfo.AuthorDate - } + case it.Type == pageparser.TypeLeadSummaryDivider: + posBody := -1 + f := func(item pageparser.Item) bool { + if posBody == -1 && !item.IsDone() { + posBody = item.Pos + } - descriptor := &pagemeta.FrontMatterDescriptor{ - Frontmatter: frontmatter, - Params: p.params, - Dates: &p.PageDates, - PageURLs: &p.URLPath, - BaseFilename: p.ContentBaseName(), - ModTime: mtime, - GitAuthorDate: gitAuthorDate, - } + if item.IsNonWhitespace() { + p.truncated = true - // Handle the date separately - // TODO(bep) we need to "do more" in this area so this can be split up and - // more easily tested without the Page, but the coupling is strong. - err := p.s.frontmatterHandler.HandleDates(descriptor) - if err != nil { - p.s.Log.ERROR.Printf("Failed to handle dates for page %q: %s", p.Path(), err) - } + // Done + return false + } + return true + } + iter.PeekWalk(f) - var draft, published, isCJKLanguage *bool - for k, v := range frontmatter { - loki := strings.ToLower(k) + p.source.posSummaryEnd = it.Pos + p.source.posBodyStart = posBody + p.source.hasSummaryDivider = true - if loki == "published" { // Intentionally undocumented - vv, err := cast.ToBoolE(v) - if err == nil { - published = &vv + if meta.markup != "html" { + // The content will be rendered by Blackfriday or similar, + // and we need to track the summary. + rn.AddReplacement(internalSummaryDividerPre, it) } - // published may also be a date - continue - } - if p.s.frontmatterHandler.IsDateKey(loki) { - continue - } + // Handle shortcode + case it.IsLeftShortcodeDelim(): + // let extractShortcode handle left delim (will do so recursively) + iter.Backup() - switch loki { - case "title": - p.title = cast.ToString(v) - p.params[loki] = p.title - case "linktitle": - p.linkTitle = cast.ToString(v) - p.params[loki] = p.linkTitle - case "description": - p.Description = cast.ToString(v) - p.params[loki] = p.Description - case "slug": - p.Slug = cast.ToString(v) - p.params[loki] = p.Slug - case "url": - if url := cast.ToString(v); strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { - return fmt.Errorf("Only relative URLs are supported, %v provided", url) - } - p.URLPath.URL = cast.ToString(v) - p.frontMatterURL = p.URLPath.URL - p.params[loki] = p.URLPath.URL - case "type": - p.contentType = cast.ToString(v) - p.params[loki] = p.contentType - case "extension", "ext": - p.extension = cast.ToString(v) - p.params[loki] = p.extension - case "keywords": - p.Keywords = cast.ToStringSlice(v) - p.params[loki] = p.Keywords - case "headless": - // For now, only the leaf bundles ("index.md") can be headless (i.e. produce no output). - // We may expand on this in the future, but that gets more complex pretty fast. - if p.TranslationBaseName() == "index" { - p.headless = cast.ToBool(v) + currShortcode, err := s.extractShortcode(ordinal, 0, iter) + if err != nil { + return fail(errors.Wrap(err, "failed to extract shortcode"), it) } - p.params[loki] = p.headless - case "outputs": - o := cast.ToStringSlice(v) - if len(o) > 0 { - // Output formats are exlicitly set in front matter, use those. - outFormats, err := p.s.outputFormatsConfig.GetByNames(o...) - - if err != nil { - p.s.Log.ERROR.Printf("Failed to resolve output formats: %s", err) - } else { - p.outputFormats = outFormats - p.params[loki] = outFormats - } - } - case "draft": - draft = new(bool) - *draft = cast.ToBool(v) - case "layout": - p.Layout = cast.ToString(v) - p.params[loki] = p.Layout - case "markup": - p.Markup = cast.ToString(v) - p.params[loki] = p.Markup - case "weight": - p.Weight = cast.ToInt(v) - p.params[loki] = p.Weight - case "aliases": - p.Aliases = cast.ToStringSlice(v) - for _, alias := range p.Aliases { - if strings.HasPrefix(alias, "http://") || strings.HasPrefix(alias, "https://") { - return fmt.Errorf("Only relative aliases are supported, %v provided", alias) - } - } - p.params[loki] = p.Aliases - case "status": - p.Status = cast.ToString(v) - p.params[loki] = p.Status - case "sitemap": - p.Sitemap = parseSitemap(cast.ToStringMap(v)) - p.params[loki] = p.Sitemap - case "iscjklanguage": - isCJKLanguage = new(bool) - *isCJKLanguage = cast.ToBool(v) - case "translationkey": - p.translationKey = cast.ToString(v) - p.params[loki] = p.translationKey - case "resources": - var resources []map[string]interface{} - handled := true - - switch vv := v.(type) { - case []map[interface{}]interface{}: - for _, vvv := range vv { - resources = append(resources, cast.ToStringMap(vvv)) - } - case []map[string]interface{}: - resources = append(resources, vv...) - case []interface{}: - for _, vvv := range vv { - switch vvvv := vvv.(type) { - case map[interface{}]interface{}: - resources = append(resources, cast.ToStringMap(vvvv)) - case map[string]interface{}: - resources = append(resources, vvvv) - } - } - default: - handled = false + currShortcode.pos = it.Pos + currShortcode.length = iter.Current().Pos - it.Pos + if currShortcode.placeholder == "" { + currShortcode.placeholder = createShortcodePlaceholder("s", currShortcode.ordinal) } - if handled { - p.params[loki] = resources - p.resourcesMetadata = resources - break + if currShortcode.name != "" { + s.nameSet[currShortcode.name] = true } - fallthrough - default: - // If not one of the explicit values, store in Params - switch vv := v.(type) { - case bool: - p.params[loki] = vv - case string: - p.params[loki] = vv - case int64, int32, int16, int8, int: - p.params[loki] = vv - case float64, float32: - p.params[loki] = vv - case time.Time: - p.params[loki] = vv - default: // handle array of strings as well - switch vvv := vv.(type) { - case []interface{}: - if len(vvv) > 0 { - switch vvv[0].(type) { - case map[interface{}]interface{}: // Proper parsing structured array from YAML based FrontMatter - p.params[loki] = vvv - case map[string]interface{}: // Proper parsing structured array from JSON based FrontMatter - p.params[loki] = vvv - case []interface{}: - p.params[loki] = vvv - default: - a := make([]string, len(vvv)) - for i, u := range vvv { - a[i] = cast.ToString(u) - } - - p.params[loki] = a - } - } else { - p.params[loki] = []string{} - } - default: - p.params[loki] = vv - } + if currShortcode.params == nil { + var s []string + currShortcode.params = s } - } - } - // Try markup explicitly set in the frontmatter - p.Markup = helpers.GuessType(p.Markup) - if p.Markup == "unknown" { - // Fall back to file extension (might also return "unknown") - p.Markup = helpers.GuessType(p.Ext()) - } + currShortcode.placeholder = createShortcodePlaceholder("s", ordinal) + ordinal++ + s.shortcodes = append(s.shortcodes, currShortcode) - if draft != nil && published != nil { - p.Draft = *draft - p.s.Log.WARN.Printf("page %q has both draft and published settings in its frontmatter. Using draft.", p.Filename()) - } else if draft != nil { - p.Draft = *draft - } else if published != nil { - p.Draft = !*published - } - p.params["draft"] = p.Draft + rn.AddShortcode(currShortcode) - if isCJKLanguage != nil { - p.isCJKLanguage = *isCJKLanguage - } else if p.s.Cfg.GetBool("hasCJKLanguage") { - if cjk.Match(p.source.parsed.Input()) { - p.isCJKLanguage = true - } else { - p.isCJKLanguage = false + case it.Type == pageparser.TypeEmoji: + if emoji := helpers.Emoji(it.ValStr()); emoji != nil { + rn.AddReplacement(emoji, it) + } else { + rn.AddBytes(it) + } + case it.IsEOF(): + break Loop + case it.IsError(): + err := fail(errors.WithStack(errors.New(it.ValStr())), it) + currShortcode.err = err + return err + + default: + rn.AddBytes(it) } } - p.params["iscjklanguage"] = p.isCJKLanguage - return nil -} + p.cmap = rn -func (p *Page) GetParam(key string) interface{} { - return p.getParam(key, false) -} - -func (p *Page) getParamToLower(key string) interface{} { - return p.getParam(key, true) + return nil } -func (p *Page) getParam(key string, stringToLower bool) interface{} { - v := p.params[strings.ToLower(key)] - - if v == nil { - return nil +func (p *pageState) errorf(err error, format string, a ...interface{}) error { + if herrors.UnwrapErrorWithFileContext(err) != nil { + // More isn't always better. + return err } - - switch val := v.(type) { - case bool: - return val - case string: - if stringToLower { - return strings.ToLower(val) - } - return val - case int64, int32, int16, int8, int: - return cast.ToInt(v) - case float64, float32: - return cast.ToFloat64(v) - case time.Time: - return val - case []string: - if stringToLower { - return helpers.SliceToLower(val) - } - return v - case map[string]interface{}: // JSON and TOML - return v - case map[interface{}]interface{}: // YAML - return v + args := append([]interface{}{p.Language().Lang, p.pathOrTitle()}, a...) + format = "[%s] page %q: " + format + if err == nil { + errors.Errorf(format, args...) + return fmt.Errorf(format, args...) } - - p.s.Log.ERROR.Printf("GetParam(\"%s\"): Unknown type %s\n", key, reflect.TypeOf(v)) - return nil + return errors.Wrapf(err, format, args...) } -func (p *Page) HasMenuCurrent(menuID string, me *MenuEntry) bool { - - sectionPagesMenu := p.Site.sectionPagesMenu - - // page is labeled as "shadow-member" of the menu with the same identifier as the section - if sectionPagesMenu != "" { - section := p.Section() - - if section != "" && sectionPagesMenu == menuID && section == me.Identifier { - return true - } +func (p *pageState) outputFormat() (f output.Format) { + if p.pageOutput == nil { + panic("no pageOutput") } + return p.pageOutput.f +} - if !me.HasChildren() { - return false +func (p *pageState) parseError(err error, input []byte, offset int) error { + if herrors.UnwrapFileError(err) != nil { + // Use the most specific location. + return err } + pos := p.posFromInput(input, offset) + return herrors.NewFileError("md", -1, pos.LineNumber, pos.ColumnNumber, err) - menus := p.Menus() - - if m, ok := menus[menuID]; ok { - - for _, child := range me.Children { - if child.IsEqual(m) { - return true - } - if p.HasMenuCurrent(menuID, child) { - return true - } - } +} +func (p *pageState) pathOrTitle() string { + if p.File() != nil { + return p.File().Filename() } - if p.IsPage() { - return false + if p.Path() != "" { + return p.Path() } - // The following logic is kept from back when Hugo had both Page and Node types. - // TODO(bep) consolidate / clean - nme := MenuEntry{Page: p, Name: p.title, URL: p.URL()} + return p.Title() +} - for _, child := range me.Children { - if nme.IsSameResource(child) { - return true - } - if p.HasMenuCurrent(menuID, child) { - return true - } - } +func (p *pageState) posFromPage(offset int) text.Position { + return p.posFromInput(p.source.parsed.Input(), offset) +} - return false +func (p *pageState) posFromInput(input []byte, offset int) text.Position { + lf := []byte("\n") + input = input[:offset] + lineNumber := bytes.Count(input, lf) + 1 + endOfLastLine := bytes.LastIndex(input, lf) + return text.Position{ + Filename: p.pathOrTitle(), + LineNumber: lineNumber, + ColumnNumber: offset - endOfLastLine, + Offset: offset, + } } -func (p *Page) IsMenuCurrent(menuID string, inme *MenuEntry) bool { - - menus := p.Menus() +func (p *pageState) posOffset(offset int) text.Position { + return p.posFromInput(p.source.parsed.Input(), offset) +} - if me, ok := menus[menuID]; ok { - if me.IsEqual(inme) { - return true - } +// shiftToOutputFormat is serialized. The output format idx refers to the +// full set of output formats for all sites. +func (p *pageState) shiftToOutputFormat(idx int) error { + if err := p.initPage(); err != nil { + return err } - if p.IsPage() { - return false + if idx >= len(p.pageOutputs) { + panic(fmt.Sprintf("invalid page state for %q: got output format index %d, have %d", p.pathOrTitle(), idx, len(p.pageOutputs))) } - // The following logic is kept from back when Hugo had both Page and Node types. - // TODO(bep) consolidate / clean - me := MenuEntry{Page: p, Name: p.title, URL: p.URL()} + p.pageOutput = p.pageOutputs[idx] - if !me.IsSameResource(inme) { - return false + if p.pageOutput == nil { + panic(fmt.Sprintf("pageOutput is nil for output idx %d", idx)) } - // this resource may be included in several menus - // search for it to make sure that it is in the menu with the given menuId - if menu, ok := (*p.Site.Menus)[menuID]; ok { - for _, menuEntry := range *menu { - if menuEntry.IsSameResource(inme) { - return true - } - - descendantFound := p.isSameAsDescendantMenu(inme, menuEntry) - if descendantFound { - return descendantFound + if idx > 0 { + // Check if we can reuse content from one of the previous formats. + for i := idx - 1; i >= 0; i-- { + po := p.pageOutputs[i] + if po.cp != nil && po.cp.reuse { + p.pageOutput.cp = po.cp + break } - } } - return false -} - -func (p *Page) isSameAsDescendantMenu(inme *MenuEntry, parent *MenuEntry) bool { - if parent.HasChildren() { - for _, child := range parent.Children { - if child.IsSameResource(inme) { - return true - } - descendantFound := p.isSameAsDescendantMenu(inme, child) - if descendantFound { - return descendantFound - } + for _, r := range p.Resources().ByType(pageResourceType) { + rp := r.(*pageState) + if err := rp.shiftToOutputFormat(idx); err != nil { + return errors.Wrap(err, "failed to shift outputformat in Page resource") } } - return false -} - -func (p *Page) Menus() PageMenus { - p.pageMenusInit.Do(func() { - p.pageMenus = PageMenus{} - - ms, ok := p.params["menus"] - if !ok { - ms, ok = p.params["menu"] - } - - if ok { - link := p.RelPermalink() - - me := MenuEntry{Page: p, Name: p.LinkTitle(), Weight: p.Weight, URL: link} - - // Could be the name of the menu to attach it to - mname, err := cast.ToStringE(ms) - - if err == nil { - me.Menu = mname - p.pageMenus[mname] = &me - return - } - - // Could be a slice of strings - mnames, err := cast.ToStringSliceE(ms) - - if err == nil { - for _, mname := range mnames { - me.Menu = mname - p.pageMenus[mname] = &me - } - return - } - - // Could be a structured menu entry - menus, err := cast.ToStringMapE(ms) - if err != nil { - p.s.Log.ERROR.Printf("unable to process menus for %q\n", p.title) - } - - for name, menu := range menus { - menuEntry := MenuEntry{Page: p, Name: p.LinkTitle(), URL: link, Weight: p.Weight, Menu: name} - if menu != nil { - p.s.Log.DEBUG.Printf("found menu: %q, in %q\n", name, p.title) - ime, err := cast.ToStringMapE(menu) - if err != nil { - p.s.Log.ERROR.Printf("unable to process menus for %q: %s", p.title, err) - } + return nil +} - menuEntry.marshallMap(ime) - } - p.pageMenus[name] = &menuEntry +func (p *pageState) getTaxonomyNodeInfo() *taxonomyNodeInfo { + info := p.s.taxonomyNodes.Get(p.SectionsEntries()...) - } - } - }) + if info == nil { + // This should never happpen + panic(fmt.Sprintf("invalid taxonomy state for %q with sections %v", p.pathOrTitle(), p.SectionsEntries())) + } - return p.pageMenus -} + return info -func (p *Page) shouldRenderTo(f output.Format) bool { - _, found := p.outputFormats.GetByName(f.Name) - return found } -// RawContent returns the un-rendered source content without -// any leading front matter. -func (p *Page) RawContent() string { - if p.source.posMainContent == -1 { - return "" +func (p *pageState) sortParentSections() { + if p.parent == nil { + return } - return string(p.source.parsed.Input()[p.source.posMainContent:]) -} - -func (p *Page) FullFilePath() string { - return filepath.Join(p.Dir(), p.LogicalName()) + page.SortByDefault(p.parent.subSections) } -// Returns the canonical, absolute fully-qualifed logical reference used by -// methods such as GetPage and ref/relref shortcodes to refer to +// sourceRef returns the reference used by GetPage and ref/relref shortcodes to refer to // this page. It is prefixed with a "/". // // For pages that have a source file, it is returns the path to this file as an // absolute path rooted in this site's content dir. // For pages that do not (sections witout content page etc.), it returns the // virtual path, consistent with where you would add a source file. -func (p *Page) absoluteSourceRef() string { - if p.File != nil { - sourcePath := p.Path() +func (p *pageState) sourceRef() string { + if p.File() != nil { + sourcePath := p.File().Path() if sourcePath != "" { return "/" + filepath.ToSlash(sourcePath) } } - if len(p.sections) > 0 { + if len(p.SectionsEntries()) > 0 { // no backing file, return the virtual source path - return "/" + path.Join(p.sections...) + return "/" + p.SectionsPath() } return "" } -// Pre render prepare steps - -func (p *Page) prepareLayouts() error { - // TODO(bep): Check the IsRenderable logic. - if p.Kind == KindPage { - if !p.IsRenderable() { - self := "__" + p.UniqueID() - err := p.s.TemplateHandler().AddLateTemplate(self, string(p.content())) - if err != nil { - return err - } - p.selfLayout = self - } - } - - return nil -} - -func (p *Page) prepareData(s *Site) error { - if p.Kind != KindSection { - var pages Pages - p.data = make(map[string]interface{}) - - switch p.Kind { - case KindPage: - case KindHome: - pages = s.RegularPages - case KindTaxonomy: - plural := p.sections[0] - term := p.sections[1] - - if s.Info.preserveTaxonomyNames { - if v, ok := s.taxonomiesOrigKey[fmt.Sprintf("%s-%s", plural, term)]; ok { - term = v - } - } - - singular := s.taxonomiesPluralSingular[plural] - taxonomy := s.Taxonomies[plural].Get(term) - - p.data[singular] = taxonomy - p.data["Singular"] = singular - p.data["Plural"] = plural - p.data["Term"] = term - pages = taxonomy.Pages() - case KindTaxonomyTerm: - plural := p.sections[0] - singular := s.taxonomiesPluralSingular[plural] - - p.data["Singular"] = singular - p.data["Plural"] = plural - p.data["Terms"] = s.Taxonomies[plural] - // keep the following just for legacy reasons - p.data["OrderedIndex"] = p.data["Terms"] - p.data["Index"] = p.data["Terms"] - - // A list of all KindTaxonomy pages with matching plural - for _, p := range s.findPagesByKind(KindTaxonomy) { - if p.sections[0] == plural { - pages = append(pages, p) - } - } - } - - p.data["Pages"] = pages - p.Pages = pages - } - - // Now we know enough to set missing dates on home page etc. - p.updatePageDates() - - return nil -} - -func (p *Page) updatePageDates() { - // TODO(bep) there is a potential issue with page sorting for home pages - // etc. without front matter dates set, but let us wrap the head around - // that in another time. - if !p.IsNode() { - return - } +type pageStatePages []*pageState - if !p.Date.IsZero() { - if p.Lastmod.IsZero() { - p.Lastmod = p.Date - } - return - } else if !p.Lastmod.IsZero() { - if p.Date.IsZero() { - p.Date = p.Lastmod - } - return - } +// Implement sorting. +func (ps pageStatePages) Len() int { return len(ps) } - // Set it to the first non Zero date in children - var foundDate, foundLastMod bool +func (ps pageStatePages) Less(i, j int) bool { return page.DefaultPageSort(ps[i], ps[j]) } - for _, child := range p.Pages { - if !child.Date.IsZero() { - p.Date = child.Date - foundDate = true - } - if !child.Lastmod.IsZero() { - p.Lastmod = child.Lastmod - foundLastMod = true - } +func (ps pageStatePages) Swap(i, j int) { ps[i], ps[j] = ps[j], ps[i] } - if foundDate && foundLastMod { - break +// findPagePos Given a page, it will find the position in Pages +// will return -1 if not found +func (ps pageStatePages) findPagePos(page *pageState) int { + for i, x := range ps { + if x.File().Filename() == page.File().Filename() { + return i } } + return -1 } -// copy creates a copy of this page with the lazy sync.Once vars reset -// so they will be evaluated again, for word count calculations etc. -func (p *Page) copy(initContent bool) *Page { - p.contentInitMu.Lock() - c := *p - p.contentInitMu.Unlock() - c.pageInit = &pageInit{} - if initContent { - if len(p.outputFormats) < 2 { - panic(fmt.Sprintf("programming error: page %q should not need to rebuild content as it has only %d outputs", p.Path(), len(p.outputFormats))) +func (ps pageStatePages) findPagePosByFilename(filename string) int { + for i, x := range ps { + if x.File().Filename() == filename { + return i } - c.pageContentInit = &pageContentInit{} } - return &c -} - -func (p *Page) Hugo() hugo.Info { - return p.s.Info.hugoInfo -} - -// GetPage looks up a page for the given ref. -// {{ with .GetPage "blog" }}{{ .Title }}{{ end }} -// -// This will return nil when no page could be found, and will return -// an error if the ref is ambiguous. -func (p *Page) GetPage(ref string) (*Page, error) { - return p.s.getPageNew(p, ref) -} - -func (p *Page) String() string { - if sourceRef := p.absoluteSourceRef(); sourceRef != "" { - return fmt.Sprintf("Page(%s)", sourceRef) - } - return fmt.Sprintf("Page(%q)", p.title) -} - -// Scratch returns the writable context associated with this Page. -func (p *Page) Scratch() *maps.Scratch { - if p.scratch == nil { - p.scratch = maps.NewScratch() - } - return p.scratch -} - -func (p *Page) Language() *langs.Language { - p.initLanguage() - return p.language -} - -func (p *Page) Lang() string { - // When set, Language can be different from lang in the case where there is a - // content file (doc.sv.md) with language indicator, but there is no language - // config for that language. Then the language will fall back on the site default. - if p.Language() != nil { - return p.Language().Lang - } - return p.lang + return -1 } -func (p *Page) isNewTranslation(candidate *Page) bool { - - if p.Kind != candidate.Kind { - return false - } - - if p.Kind == KindPage || p.Kind == kindUnknown { - panic("Node type not currently supported for this op") - } - - // At this point, we know that this is a traditional Node (home page, section, taxonomy) - // It represents the same node, but different language, if the sections is the same. - if len(p.sections) != len(candidate.sections) { - return false +func (ps pageStatePages) findPagePosByFilnamePrefix(prefix string) int { + if prefix == "" { + return -1 } - for i := 0; i < len(p.sections); i++ { - if p.sections[i] != candidate.sections[i] { - return false - } - } + lenDiff := -1 + currPos := -1 + prefixLen := len(prefix) - // Finally check that it is not already added. - for _, translation := range p.translations { - if candidate == translation { - return false + // Find the closest match + for i, x := range ps { + if strings.HasPrefix(x.File().Filename(), prefix) { + diff := len(x.File().Filename()) - prefixLen + if lenDiff == -1 || diff < lenDiff { + lenDiff = diff + currPos = i + } } } - - return true - -} - -func (p *Page) shouldAddLanguagePrefix() bool { - if !p.Site.IsMultiLingual() { - return false - } - - if p.s.owner.IsMultihost() { - return true - } - - if p.Lang() == "" { - return false - } - - if !p.Site.defaultContentLanguageInSubdir && p.Lang() == p.s.multilingual().DefaultLang.Lang { - return false - } - - return true -} - -func (p *Page) initLanguage() { - p.languageInit.Do(func() { - if p.language != nil { - return - } - - ml := p.s.multilingual() - if ml == nil { - panic("Multilanguage not set") - } - if p.lang == "" { - p.lang = ml.DefaultLang.Lang - p.language = ml.DefaultLang - return - } - - language := ml.Language(p.lang) - - if language == nil { - language = ml.DefaultLang - } - - p.language = language - - }) -} - -func (p *Page) LanguagePrefix() string { - return p.Site.LanguagePrefix + return currPos } -func (p *Page) addLangPathPrefixIfFlagSet(outfile string, should bool) string { - if helpers.IsAbsURL(outfile) { - return outfile - } - - if !should { - return outfile +func content(c resource.ContentProvider) string { + cc, err := c.Content() + if err != nil { + panic(err) } - hadSlashSuffix := strings.HasSuffix(outfile, "/") - - outfile = "/" + path.Join(p.Lang(), outfile) - if hadSlashSuffix { - outfile += "/" + ccs, err := cast.ToStringE(cc) + if err != nil { + panic(err) } - return outfile + return ccs } -func sectionsFromFile(fi *fileInfo) []string { +func (s *Site) sectionsFromFile(fi source.File) []string { dirname := fi.Dir() dirname = strings.Trim(dirname, helpers.FilePathSeparator) if dirname == "" { @@ -2028,96 +852,18 @@ func sectionsFromFile(fi *fileInfo) []string { } parts := strings.Split(dirname, helpers.FilePathSeparator) - if fi.bundleTp == bundleLeaf && len(parts) > 0 { - // my-section/mybundle/index.md => my-section - return parts[:len(parts)-1] - } - - return parts -} - -func kindFromFileInfo(fi *fileInfo) string { - if fi.TranslationBaseName() == "_index" { - if fi.Dir() == "" { - return KindHome + if fii, ok := fi.(*fileInfo); ok { + if fii.bundleTp == bundleLeaf && len(parts) > 0 { + // my-section/mybundle/index.md => my-section + return parts[:len(parts)-1] } - // Could be index for section, taxonomy, taxonomy term - // We don't know enough yet to determine which - return kindUnknown - } - return KindPage -} - -func (p *Page) sectionsPath() string { - if len(p.sections) == 0 { - return "" } - if len(p.sections) == 1 { - return p.sections[0] - } - - return path.Join(p.sections...) -} - -func (p *Page) kindFromSections() string { - if len(p.sections) == 0 || len(p.s.Taxonomies) == 0 { - return KindSection - } - - sectionPath := p.sectionsPath() - - for k, _ := range p.s.Taxonomies { - if k == sectionPath { - return KindTaxonomyTerm - } - if strings.HasPrefix(sectionPath, k) { - return KindTaxonomy - } - } - - return KindSection -} - -func (p *Page) setValuesForKind(s *Site) { - if p.Kind == kindUnknown { - // This is either a taxonomy list, taxonomy term or a section - nodeType := p.kindFromSections() - - if nodeType == kindUnknown { - panic(fmt.Sprintf("Unable to determine page kind from %q", p.sections)) - } - - p.Kind = nodeType - } - - switch p.Kind { - case KindHome: - p.URLPath.URL = "/" - case KindPage: - default: - if p.URLPath.URL == "" { - p.URLPath.URL = "/" + path.Join(p.sections...) + "/" - } - } -} - -// Used in error logs. -func (p *Page) pathOrTitle() string { - if p.Filename() != "" { - return p.Filename() - } - return p.title -} - -func (p *Page) Next() *Page { - // TODO Remove the deprecation notice (but keep PrevPage as an alias) Hugo 0.52 - helpers.Deprecated("Page", ".Next", "Use .PrevPage (yes, not .NextPage).", false) - return p.PrevPage + return parts } -func (p *Page) Prev() *Page { - // TODO Remove the deprecation notice (but keep NextPage as an alias) Hugo 0.52 - helpers.Deprecated("Page", ".Prev", "Use .NextPage (yes, not .PrevPage).", false) - return p.NextPage +func printStackTrace(length int) string { + trace := make([]byte, length) + runtime.Stack(trace, true) + return string(trace) } diff --git a/hugolib/pageCache.go b/hugolib/pageCache.go deleted file mode 100644 index 485da4ba3..000000000 --- a/hugolib/pageCache.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "sync" -) - -type pageCacheEntry struct { - in []Pages - out Pages -} - -func (entry pageCacheEntry) matches(pageLists []Pages) bool { - if len(entry.in) != len(pageLists) { - return false - } - for i, p := range pageLists { - if !pagesEqual(p, entry.in[i]) { - return false - } - } - - return true -} - -type pageCache struct { - sync.RWMutex - m map[string][]pageCacheEntry -} - -func newPageCache() *pageCache { - return &pageCache{m: make(map[string][]pageCacheEntry)} -} - -func (c *pageCache) clear() { - c.Lock() - defer c.Unlock() - c.m = make(map[string][]pageCacheEntry) -} - -// get/getP gets a Pages slice from the cache matching the given key and -// all the provided Pages slices. -// If none found in cache, a copy of the first slice is created. -// -// If an apply func is provided, that func is applied to the newly created copy. -// -// The getP variant' apply func takes a pointer to Pages. -// -// The cache and the execution of the apply func is protected by a RWMutex. -func (c *pageCache) get(key string, apply func(p Pages), pageLists ...Pages) (Pages, bool) { - return c.getP(key, func(p *Pages) { - if apply != nil { - apply(*p) - } - }, pageLists...) -} - -func (c *pageCache) getP(key string, apply func(p *Pages), pageLists ...Pages) (Pages, bool) { - c.RLock() - if cached, ok := c.m[key]; ok { - for _, entry := range cached { - if entry.matches(pageLists) { - c.RUnlock() - return entry.out, true - } - } - } - c.RUnlock() - - c.Lock() - defer c.Unlock() - - // double-check - if cached, ok := c.m[key]; ok { - for _, entry := range cached { - if entry.matches(pageLists) { - return entry.out, true - } - } - } - - p := pageLists[0] - pagesCopy := append(Pages(nil), p...) - - if apply != nil { - apply(&pagesCopy) - } - - entry := pageCacheEntry{in: pageLists, out: pagesCopy} - if v, ok := c.m[key]; ok { - c.m[key] = append(v, entry) - } else { - c.m[key] = []pageCacheEntry{entry} - } - - return pagesCopy, false - -} - -// pagesEqual returns whether p1 and p2 are equal. -func pagesEqual(p1, p2 Pages) bool { - if p1 == nil && p2 == nil { - return true - } - - if p1 == nil || p2 == nil { - return false - } - - if p1.Len() != p2.Len() { - return false - } - - if p1.Len() == 0 { - return true - } - - for i := 0; i < len(p1); i++ { - if p1[i] != p2[i] { - return false - } - } - return true -} diff --git a/hugolib/pageCache_test.go b/hugolib/pageCache_test.go deleted file mode 100644 index 48f595f86..000000000 --- a/hugolib/pageCache_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "strconv" - "sync" - "sync/atomic" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestPageCache(t *testing.T) { - t.Parallel() - c1 := newPageCache() - - changeFirst := func(p Pages) { - p[0].Description = "changed" - } - - var o1 uint64 - var o2 uint64 - - var wg sync.WaitGroup - - var l1 sync.Mutex - var l2 sync.Mutex - - var testPageSets []Pages - - s := newTestSite(t) - - for i := 0; i < 50; i++ { - testPageSets = append(testPageSets, createSortTestPages(s, i+1)) - } - - for j := 0; j < 100; j++ { - wg.Add(1) - go func() { - defer wg.Done() - for k, pages := range testPageSets { - l1.Lock() - p, c := c1.get("k1", nil, pages) - assert.Equal(t, !atomic.CompareAndSwapUint64(&o1, uint64(k), uint64(k+1)), c) - l1.Unlock() - p2, c2 := c1.get("k1", nil, p) - assert.True(t, c2) - assert.True(t, pagesEqual(p, p2)) - assert.True(t, pagesEqual(p, pages)) - assert.NotNil(t, p) - - l2.Lock() - p3, c3 := c1.get("k2", changeFirst, pages) - assert.Equal(t, !atomic.CompareAndSwapUint64(&o2, uint64(k), uint64(k+1)), c3) - l2.Unlock() - assert.NotNil(t, p3) - assert.Equal(t, p3[0].Description, "changed") - } - }() - } - wg.Wait() -} - -func BenchmarkPageCache(b *testing.B) { - cache := newPageCache() - pages := make(Pages, 30) - for i := 0; i < 30; i++ { - pages[i] = &Page{title: "p" + strconv.Itoa(i)} - } - key := "key" - - b.ResetTimer() - for i := 0; i < b.N; i++ { - cache.getP(key, nil, pages) - } -} diff --git a/hugolib/pageGroup.go b/hugolib/pageGroup.go deleted file mode 100644 index 8aaa1018c..000000000 --- a/hugolib/pageGroup.go +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "errors" - "reflect" - "sort" - "strings" - "time" -) - -// PageGroup represents a group of pages, grouped by the key. -// The key is typically a year or similar. -type PageGroup struct { - Key interface{} - Pages -} - -type mapKeyValues []reflect.Value - -func (v mapKeyValues) Len() int { return len(v) } -func (v mapKeyValues) Swap(i, j int) { v[i], v[j] = v[j], v[i] } - -type mapKeyByInt struct{ mapKeyValues } - -func (s mapKeyByInt) Less(i, j int) bool { return s.mapKeyValues[i].Int() < s.mapKeyValues[j].Int() } - -type mapKeyByStr struct{ mapKeyValues } - -func (s mapKeyByStr) Less(i, j int) bool { - return s.mapKeyValues[i].String() < s.mapKeyValues[j].String() -} - -func sortKeys(v []reflect.Value, order string) []reflect.Value { - if len(v) <= 1 { - return v - } - - switch v[0].Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - if order == "desc" { - sort.Sort(sort.Reverse(mapKeyByInt{v})) - } else { - sort.Sort(mapKeyByInt{v}) - } - case reflect.String: - if order == "desc" { - sort.Sort(sort.Reverse(mapKeyByStr{v})) - } else { - sort.Sort(mapKeyByStr{v}) - } - } - return v -} - -// PagesGroup represents a list of page groups. -// This is what you get when doing page grouping in the templates. -type PagesGroup []PageGroup - -// Reverse reverses the order of this list of page groups. -func (p PagesGroup) Reverse() PagesGroup { - for i, j := 0, len(p)-1; i < j; i, j = i+1, j-1 { - p[i], p[j] = p[j], p[i] - } - - return p -} - -var ( - errorType = reflect.TypeOf((*error)(nil)).Elem() - pagePtrType = reflect.TypeOf((*Page)(nil)) -) - -// GroupBy groups by the value in the given field or method name and with the given order. -// Valid values for order is asc, desc, rev and reverse. -func (p Pages) GroupBy(key string, order ...string) (PagesGroup, error) { - if len(p) < 1 { - return nil, nil - } - - direction := "asc" - - if len(order) > 0 && (strings.ToLower(order[0]) == "desc" || strings.ToLower(order[0]) == "rev" || strings.ToLower(order[0]) == "reverse") { - direction = "desc" - } - - var ft interface{} - m, ok := pagePtrType.MethodByName(key) - if ok { - if m.Type.NumIn() != 1 || m.Type.NumOut() == 0 || m.Type.NumOut() > 2 { - return nil, errors.New(key + " is a Page method but you can't use it with GroupBy") - } - if m.Type.NumOut() == 1 && m.Type.Out(0).Implements(errorType) { - return nil, errors.New(key + " is a Page method but you can't use it with GroupBy") - } - if m.Type.NumOut() == 2 && !m.Type.Out(1).Implements(errorType) { - return nil, errors.New(key + " is a Page method but you can't use it with GroupBy") - } - ft = m - } else { - ft, ok = pagePtrType.Elem().FieldByName(key) - if !ok { - return nil, errors.New(key + " is neither a field nor a method of Page") - } - } - - var tmp reflect.Value - switch e := ft.(type) { - case reflect.StructField: - tmp = reflect.MakeMap(reflect.MapOf(e.Type, reflect.SliceOf(pagePtrType))) - case reflect.Method: - tmp = reflect.MakeMap(reflect.MapOf(e.Type.Out(0), reflect.SliceOf(pagePtrType))) - } - - for _, e := range p { - ppv := reflect.ValueOf(e) - var fv reflect.Value - switch ft.(type) { - case reflect.StructField: - fv = ppv.Elem().FieldByName(key) - case reflect.Method: - fv = ppv.MethodByName(key).Call([]reflect.Value{})[0] - } - if !fv.IsValid() { - continue - } - if !tmp.MapIndex(fv).IsValid() { - tmp.SetMapIndex(fv, reflect.MakeSlice(reflect.SliceOf(pagePtrType), 0, 0)) - } - tmp.SetMapIndex(fv, reflect.Append(tmp.MapIndex(fv), ppv)) - } - - sortedKeys := sortKeys(tmp.MapKeys(), direction) - r := make([]PageGroup, len(sortedKeys)) - for i, k := range sortedKeys { - r[i] = PageGroup{Key: k.Interface(), Pages: tmp.MapIndex(k).Interface().([]*Page)} - } - - return r, nil -} - -// GroupByParam groups by the given page parameter key's value and with the given order. -// Valid values for order is asc, desc, rev and reverse. -func (p Pages) GroupByParam(key string, order ...string) (PagesGroup, error) { - if len(p) < 1 { - return nil, nil - } - - direction := "asc" - - if len(order) > 0 && (strings.ToLower(order[0]) == "desc" || strings.ToLower(order[0]) == "rev" || strings.ToLower(order[0]) == "reverse") { - direction = "desc" - } - - var tmp reflect.Value - var keyt reflect.Type - for _, e := range p { - param := e.getParamToLower(key) - if param != nil { - if _, ok := param.([]string); !ok { - keyt = reflect.TypeOf(param) - tmp = reflect.MakeMap(reflect.MapOf(keyt, reflect.SliceOf(pagePtrType))) - break - } - } - } - if !tmp.IsValid() { - return nil, errors.New("There is no such a param") - } - - for _, e := range p { - param := e.getParam(key, false) - if param == nil || reflect.TypeOf(param) != keyt { - continue - } - v := reflect.ValueOf(param) - if !tmp.MapIndex(v).IsValid() { - tmp.SetMapIndex(v, reflect.MakeSlice(reflect.SliceOf(pagePtrType), 0, 0)) - } - tmp.SetMapIndex(v, reflect.Append(tmp.MapIndex(v), reflect.ValueOf(e))) - } - - var r []PageGroup - for _, k := range sortKeys(tmp.MapKeys(), direction) { - r = append(r, PageGroup{Key: k.Interface(), Pages: tmp.MapIndex(k).Interface().([]*Page)}) - } - - return r, nil -} - -func (p Pages) groupByDateField(sorter func(p Pages) Pages, formatter func(p *Page) string, order ...string) (PagesGroup, error) { - if len(p) < 1 { - return nil, nil - } - - sp := sorter(p) - - if !(len(order) > 0 && (strings.ToLower(order[0]) == "asc" || strings.ToLower(order[0]) == "rev" || strings.ToLower(order[0]) == "reverse")) { - sp = sp.Reverse() - } - - date := formatter(sp[0]) - var r []PageGroup - r = append(r, PageGroup{Key: date, Pages: make(Pages, 0)}) - r[0].Pages = append(r[0].Pages, sp[0]) - - i := 0 - for _, e := range sp[1:] { - date = formatter(e) - if r[i].Key.(string) != date { - r = append(r, PageGroup{Key: date}) - i++ - } - r[i].Pages = append(r[i].Pages, e) - } - return r, nil -} - -// GroupByDate groups by the given page's Date value in -// the given format and with the given order. -// Valid values for order is asc, desc, rev and reverse. -// For valid format strings, see https://golang.org/pkg/time/#Time.Format -func (p Pages) GroupByDate(format string, order ...string) (PagesGroup, error) { - sorter := func(p Pages) Pages { - return p.ByDate() - } - formatter := func(p *Page) string { - return p.Date.Format(format) - } - return p.groupByDateField(sorter, formatter, order...) -} - -// GroupByPublishDate groups by the given page's PublishDate value in -// the given format and with the given order. -// Valid values for order is asc, desc, rev and reverse. -// For valid format strings, see https://golang.org/pkg/time/#Time.Format -func (p Pages) GroupByPublishDate(format string, order ...string) (PagesGroup, error) { - sorter := func(p Pages) Pages { - return p.ByPublishDate() - } - formatter := func(p *Page) string { - return p.PublishDate.Format(format) - } - return p.groupByDateField(sorter, formatter, order...) -} - -// GroupByExpiryDate groups by the given page's ExpireDate value in -// the given format and with the given order. -// Valid values for order is asc, desc, rev and reverse. -// For valid format strings, see https://golang.org/pkg/time/#Time.Format -func (p Pages) GroupByExpiryDate(format string, order ...string) (PagesGroup, error) { - sorter := func(p Pages) Pages { - return p.ByExpiryDate() - } - formatter := func(p *Page) string { - return p.ExpiryDate.Format(format) - } - return p.groupByDateField(sorter, formatter, order...) -} - -// GroupByParamDate groups by a date set as a param on the page in -// the given format and with the given order. -// Valid values for order is asc, desc, rev and reverse. -// For valid format strings, see https://golang.org/pkg/time/#Time.Format -func (p Pages) GroupByParamDate(key string, format string, order ...string) (PagesGroup, error) { - sorter := func(p Pages) Pages { - var r Pages - for _, e := range p { - param := e.getParamToLower(key) - if param != nil { - if _, ok := param.(time.Time); ok { - r = append(r, e) - } - } - } - pdate := func(p1, p2 *Page) bool { - return p1.getParamToLower(key).(time.Time).Unix() < p2.getParamToLower(key).(time.Time).Unix() - } - pageBy(pdate).Sort(r) - return r - } - formatter := func(p *Page) string { - return p.getParamToLower(key).(time.Time).Format(format) - } - return p.groupByDateField(sorter, formatter, order...) -} diff --git a/hugolib/pageGroup_test.go b/hugolib/pageGroup_test.go deleted file mode 100644 index febcb3c1c..000000000 --- a/hugolib/pageGroup_test.go +++ /dev/null @@ -1,457 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "errors" - "path/filepath" - "reflect" - "testing" - - "github.com/spf13/cast" -) - -type pageGroupTestObject struct { - path string - weight int - date string - param string -} - -var pageGroupTestSources = []pageGroupTestObject{ - {"/section1/testpage1.md", 3, "2012-04-06", "foo"}, - {"/section1/testpage2.md", 3, "2012-01-01", "bar"}, - {"/section1/testpage3.md", 2, "2012-04-06", "foo"}, - {"/section2/testpage4.md", 1, "2012-03-02", "bar"}, - {"/section2/testpage5.md", 1, "2012-04-06", "baz"}, -} - -func preparePageGroupTestPages(t *testing.T) Pages { - s := newTestSite(t) - var pages Pages - for _, src := range pageGroupTestSources { - p, err := s.NewPage(filepath.FromSlash(src.path)) - if err != nil { - t.Fatalf("failed to prepare test page %s", src.path) - } - p.Weight = src.weight - p.Date = cast.ToTime(src.date) - p.PublishDate = cast.ToTime(src.date) - p.ExpiryDate = cast.ToTime(src.date) - p.params["custom_param"] = src.param - p.params["custom_date"] = cast.ToTime(src.date) - pages = append(pages, p) - } - return pages -} - -func TestGroupByWithFieldNameArg(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: 1, Pages: Pages{pages[3], pages[4]}}, - {Key: 2, Pages: Pages{pages[2]}}, - {Key: 3, Pages: Pages{pages[0], pages[1]}}, - } - - groups, err := pages.GroupBy("Weight") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByWithMethodNameArg(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: "section1", Pages: Pages{pages[0], pages[1], pages[2]}}, - {Key: "section2", Pages: Pages{pages[3], pages[4]}}, - } - - groups, err := pages.GroupBy("Type") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByWithSectionArg(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: "section1", Pages: Pages{pages[0], pages[1], pages[2]}}, - {Key: "section2", Pages: Pages{pages[3], pages[4]}}, - } - - groups, err := pages.GroupBy("Section") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByInReverseOrder(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: 3, Pages: Pages{pages[0], pages[1]}}, - {Key: 2, Pages: Pages{pages[2]}}, - {Key: 1, Pages: Pages{pages[3], pages[4]}}, - } - - groups, err := pages.GroupBy("Weight", "desc") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByCalledWithEmptyPages(t *testing.T) { - t.Parallel() - var pages Pages - groups, err := pages.GroupBy("Weight") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if groups != nil { - t.Errorf("PagesGroup isn't empty. It should be %#v, got %#v", nil, groups) - } -} - -func TestGroupByCalledWithUnavailableKey(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - _, err := pages.GroupBy("UnavailableKey") - if err == nil { - t.Errorf("GroupByParam should return an error but didn't") - } -} - -func (page *Page) DummyPageMethodWithArgForTest(s string) string { - return s -} - -func (page *Page) DummyPageMethodReturnThreeValueForTest() (string, string, string) { - return "foo", "bar", "baz" -} - -func (page *Page) DummyPageMethodReturnErrorOnlyForTest() error { - return errors.New("some error occurred") -} - -func (page *Page) dummyPageMethodReturnTwoValueForTest() (string, string) { - return "foo", "bar" -} - -func TestGroupByCalledWithInvalidMethod(t *testing.T) { - t.Parallel() - var err error - pages := preparePageGroupTestPages(t) - - _, err = pages.GroupBy("DummyPageMethodWithArgForTest") - if err == nil { - t.Errorf("GroupByParam should return an error but didn't") - } - - _, err = pages.GroupBy("DummyPageMethodReturnThreeValueForTest") - if err == nil { - t.Errorf("GroupByParam should return an error but didn't") - } - - _, err = pages.GroupBy("DummyPageMethodReturnErrorOnlyForTest") - if err == nil { - t.Errorf("GroupByParam should return an error but didn't") - } - - _, err = pages.GroupBy("DummyPageMethodReturnTwoValueForTest") - if err == nil { - t.Errorf("GroupByParam should return an error but didn't") - } -} - -func TestReverse(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - - groups1, err := pages.GroupBy("Weight", "desc") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - - groups2, err := pages.GroupBy("Weight") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - groups2 = groups2.Reverse() - - if !reflect.DeepEqual(groups2, groups1) { - t.Errorf("PagesGroup is sorted in unexpected order. It should be %#v, got %#v", groups2, groups1) - } -} - -func TestGroupByParam(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: "bar", Pages: Pages{pages[1], pages[3]}}, - {Key: "baz", Pages: Pages{pages[4]}}, - {Key: "foo", Pages: Pages{pages[0], pages[2]}}, - } - - groups, err := pages.GroupByParam("custom_param") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByParamInReverseOrder(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: "foo", Pages: Pages{pages[0], pages[2]}}, - {Key: "baz", Pages: Pages{pages[4]}}, - {Key: "bar", Pages: Pages{pages[1], pages[3]}}, - } - - groups, err := pages.GroupByParam("custom_param", "desc") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByParamCalledWithCapitalLetterString(t *testing.T) { - testStr := "TestString" - f := "/section1/test_capital.md" - s := newTestSite(t) - p, err := s.NewPage(filepath.FromSlash(f)) - if err != nil { - t.Fatalf("failed to prepare test page %s", f) - } - p.params["custom_param"] = testStr - pages := Pages{p} - - groups, err := pages.GroupByParam("custom_param") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if groups[0].Key != testStr { - t.Errorf("PagesGroup key is converted to a lower character string. It should be %#v, got %#v", testStr, groups[0].Key) - } -} - -func TestGroupByParamCalledWithSomeUnavailableParams(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - delete(pages[1].params, "custom_param") - delete(pages[3].params, "custom_param") - delete(pages[4].params, "custom_param") - - expect := PagesGroup{ - {Key: "foo", Pages: Pages{pages[0], pages[2]}}, - } - - groups, err := pages.GroupByParam("custom_param") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByParamCalledWithEmptyPages(t *testing.T) { - t.Parallel() - var pages Pages - groups, err := pages.GroupByParam("custom_param") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if groups != nil { - t.Errorf("PagesGroup isn't empty. It should be %#v, got %#v", nil, groups) - } -} - -func TestGroupByParamCalledWithUnavailableParam(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - _, err := pages.GroupByParam("unavailable_param") - if err == nil { - t.Errorf("GroupByParam should return an error but didn't") - } -} - -func TestGroupByDate(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: "2012-04", Pages: Pages{pages[4], pages[2], pages[0]}}, - {Key: "2012-03", Pages: Pages{pages[3]}}, - {Key: "2012-01", Pages: Pages{pages[1]}}, - } - - groups, err := pages.GroupByDate("2006-01") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByDateInReverseOrder(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: "2012-01", Pages: Pages{pages[1]}}, - {Key: "2012-03", Pages: Pages{pages[3]}}, - {Key: "2012-04", Pages: Pages{pages[0], pages[2], pages[4]}}, - } - - groups, err := pages.GroupByDate("2006-01", "asc") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByPublishDate(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: "2012-04", Pages: Pages{pages[4], pages[2], pages[0]}}, - {Key: "2012-03", Pages: Pages{pages[3]}}, - {Key: "2012-01", Pages: Pages{pages[1]}}, - } - - groups, err := pages.GroupByPublishDate("2006-01") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByPublishDateInReverseOrder(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: "2012-01", Pages: Pages{pages[1]}}, - {Key: "2012-03", Pages: Pages{pages[3]}}, - {Key: "2012-04", Pages: Pages{pages[0], pages[2], pages[4]}}, - } - - groups, err := pages.GroupByDate("2006-01", "asc") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByPublishDateWithEmptyPages(t *testing.T) { - t.Parallel() - var pages Pages - groups, err := pages.GroupByPublishDate("2006-01") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if groups != nil { - t.Errorf("PagesGroup isn't empty. It should be %#v, got %#v", nil, groups) - } -} - -func TestGroupByExpiryDate(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: "2012-04", Pages: Pages{pages[4], pages[2], pages[0]}}, - {Key: "2012-03", Pages: Pages{pages[3]}}, - {Key: "2012-01", Pages: Pages{pages[1]}}, - } - - groups, err := pages.GroupByExpiryDate("2006-01") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByParamDate(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: "2012-04", Pages: Pages{pages[4], pages[2], pages[0]}}, - {Key: "2012-03", Pages: Pages{pages[3]}}, - {Key: "2012-01", Pages: Pages{pages[1]}}, - } - - groups, err := pages.GroupByParamDate("custom_date", "2006-01") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByParamDateInReverseOrder(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - expect := PagesGroup{ - {Key: "2012-01", Pages: Pages{pages[1]}}, - {Key: "2012-03", Pages: Pages{pages[3]}}, - {Key: "2012-04", Pages: Pages{pages[0], pages[2], pages[4]}}, - } - - groups, err := pages.GroupByParamDate("custom_date", "2006-01", "asc") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if !reflect.DeepEqual(groups, expect) { - t.Errorf("PagesGroup has unexpected groups. It should be %#v, got %#v", expect, groups) - } -} - -func TestGroupByParamDateWithEmptyPages(t *testing.T) { - t.Parallel() - var pages Pages - groups, err := pages.GroupByParamDate("custom_date", "2006-01") - if err != nil { - t.Fatalf("Unable to make PagesGroup array: %s", err) - } - if groups != nil { - t.Errorf("PagesGroup isn't empty. It should be %#v, got %#v", nil, groups) - } -} diff --git a/hugolib/pageSort.go b/hugolib/pageSort.go deleted file mode 100644 index 454beb473..000000000 --- a/hugolib/pageSort.go +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "github.com/gohugoio/hugo/helpers" - - "sort" - - "github.com/spf13/cast" -) - -var spc = newPageCache() - -/* - * Implementation of a custom sorter for Pages - */ - -// A pageSorter implements the sort interface for Pages -type pageSorter struct { - pages Pages - by pageBy -} - -// pageBy is a closure used in the Sort.Less method. -type pageBy func(p1, p2 *Page) bool - -// Sort stable sorts the pages given the receiver's sort order. -func (by pageBy) Sort(pages Pages) { - ps := &pageSorter{ - pages: pages, - by: by, // The Sort method's receiver is the function (closure) that defines the sort order. - } - sort.Stable(ps) -} - -// defaultPageSort is the default sort for pages in Hugo: -// Order by Weight, Date, LinkTitle and then full file path. -var defaultPageSort = func(p1, p2 *Page) bool { - if p1.Weight == p2.Weight { - if p1.Date.Unix() == p2.Date.Unix() { - if p1.LinkTitle() == p2.LinkTitle() { - return (p1.FullFilePath() < p2.FullFilePath()) - } - return (p1.LinkTitle() < p2.LinkTitle()) - } - return p1.Date.Unix() > p2.Date.Unix() - } - - if p2.Weight == 0 { - return true - } - - if p1.Weight == 0 { - return false - } - - return p1.Weight < p2.Weight -} - -var languagePageSort = func(p1, p2 *Page) bool { - if p1.Language().Weight == p2.Language().Weight { - if p1.Date.Unix() == p2.Date.Unix() { - if p1.LinkTitle() == p2.LinkTitle() { - return (p1.FullFilePath() < p2.FullFilePath()) - } - return (p1.LinkTitle() < p2.LinkTitle()) - } - return p1.Date.Unix() > p2.Date.Unix() - } - - if p2.Language().Weight == 0 { - return true - } - - if p1.Language().Weight == 0 { - return false - } - - return p1.Language().Weight < p2.Language().Weight -} - -func (ps *pageSorter) Len() int { return len(ps.pages) } -func (ps *pageSorter) Swap(i, j int) { ps.pages[i], ps.pages[j] = ps.pages[j], ps.pages[i] } - -// Less is part of sort.Interface. It is implemented by calling the "by" closure in the sorter. -func (ps *pageSorter) Less(i, j int) bool { return ps.by(ps.pages[i], ps.pages[j]) } - -// Sort sorts the pages by the default sort order defined: -// Order by Weight, Date, LinkTitle and then full file path. -func (p Pages) Sort() { - // Remove in Hugo 0.51 - helpers.Deprecated("Pages", "Sort", "Use .ByWeight", true) - p.sort() -} - -func (p Pages) sort() { - pageBy(defaultPageSort).Sort(p) -} - -// Limit limits the number of pages returned to n. -func (p Pages) Limit(n int) Pages { - if len(p) > n { - return p[0:n] - } - return p -} - -// ByWeight sorts the Pages by weight and returns a copy. -// -// Adjacent invocations on the same receiver will return a cached result. -// -// This may safely be executed in parallel. -func (p Pages) ByWeight() Pages { - const key = "pageSort.ByWeight" - pages, _ := spc.get(key, pageBy(defaultPageSort).Sort, p) - return pages -} - -// ByTitle sorts the Pages by title and returns a copy. -// -// Adjacent invocations on the same receiver will return a cached result. -// -// This may safely be executed in parallel. -func (p Pages) ByTitle() Pages { - - const key = "pageSort.ByTitle" - - title := func(p1, p2 *Page) bool { - return p1.title < p2.title - } - - pages, _ := spc.get(key, pageBy(title).Sort, p) - return pages -} - -// ByLinkTitle sorts the Pages by link title and returns a copy. -// -// Adjacent invocations on the same receiver will return a cached result. -// -// This may safely be executed in parallel. -func (p Pages) ByLinkTitle() Pages { - - const key = "pageSort.ByLinkTitle" - - linkTitle := func(p1, p2 *Page) bool { - return p1.LinkTitle() < p2.LinkTitle() - } - - pages, _ := spc.get(key, pageBy(linkTitle).Sort, p) - - return pages -} - -// ByDate sorts the Pages by date and returns a copy. -// -// Adjacent invocations on the same receiver will return a cached result. -// -// This may safely be executed in parallel. -func (p Pages) ByDate() Pages { - - const key = "pageSort.ByDate" - - date := func(p1, p2 *Page) bool { - return p1.Date.Unix() < p2.Date.Unix() - } - - pages, _ := spc.get(key, pageBy(date).Sort, p) - - return pages -} - -// ByPublishDate sorts the Pages by publish date and returns a copy. -// -// Adjacent invocations on the same receiver will return a cached result. -// -// This may safely be executed in parallel. -func (p Pages) ByPublishDate() Pages { - - const key = "pageSort.ByPublishDate" - - pubDate := func(p1, p2 *Page) bool { - return p1.PublishDate.Unix() < p2.PublishDate.Unix() - } - - pages, _ := spc.get(key, pageBy(pubDate).Sort, p) - - return pages -} - -// ByExpiryDate sorts the Pages by publish date and returns a copy. -// -// Adjacent invocations on the same receiver will return a cached result. -// -// This may safely be executed in parallel. -func (p Pages) ByExpiryDate() Pages { - - const key = "pageSort.ByExpiryDate" - - expDate := func(p1, p2 *Page) bool { - return p1.ExpiryDate.Unix() < p2.ExpiryDate.Unix() - } - - pages, _ := spc.get(key, pageBy(expDate).Sort, p) - - return pages -} - -// ByLastmod sorts the Pages by the last modification date and returns a copy. -// -// Adjacent invocations on the same receiver will return a cached result. -// -// This may safely be executed in parallel. -func (p Pages) ByLastmod() Pages { - - const key = "pageSort.ByLastmod" - - date := func(p1, p2 *Page) bool { - return p1.Lastmod.Unix() < p2.Lastmod.Unix() - } - - pages, _ := spc.get(key, pageBy(date).Sort, p) - - return pages -} - -// ByLength sorts the Pages by length and returns a copy. -// -// Adjacent invocations on the same receiver will return a cached result. -// -// This may safely be executed in parallel. -func (p Pages) ByLength() Pages { - - const key = "pageSort.ByLength" - - length := func(p1, p2 *Page) bool { - return len(p1.content()) < len(p2.content()) - } - - pages, _ := spc.get(key, pageBy(length).Sort, p) - - return pages -} - -// ByLanguage sorts the Pages by the language's Weight. -// -// Adjacent invocations on the same receiver will return a cached result. -// -// This may safely be executed in parallel. -func (p Pages) ByLanguage() Pages { - - const key = "pageSort.ByLanguage" - - pages, _ := spc.get(key, pageBy(languagePageSort).Sort, p) - - return pages -} - -// Reverse reverses the order in Pages and returns a copy. -// -// Adjacent invocations on the same receiver will return a cached result. -// -// This may safely be executed in parallel. -func (p Pages) Reverse() Pages { - const key = "pageSort.Reverse" - - reverseFunc := func(pages Pages) { - for i, j := 0, len(pages)-1; i < j; i, j = i+1, j-1 { - pages[i], pages[j] = pages[j], pages[i] - } - } - - pages, _ := spc.get(key, reverseFunc, p) - - return pages -} - -// ByParam sorts the pages according to the given page Params key. -// -// Adjacent invocations on the same receiver with the same paramsKey will return a cached result. -// -// This may safely be executed in parallel. -func (p Pages) ByParam(paramsKey interface{}) Pages { - paramsKeyStr := cast.ToString(paramsKey) - key := "pageSort.ByParam." + paramsKeyStr - - paramsKeyComparator := func(p1, p2 *Page) bool { - v1, _ := p1.Param(paramsKeyStr) - v2, _ := p2.Param(paramsKeyStr) - - if v1 == nil { - return false - } - - if v2 == nil { - return true - } - - isNumeric := func(v interface{}) bool { - switch v.(type) { - case uint8, uint16, uint32, uint64, int, int8, int16, int32, int64, float32, float64: - return true - default: - return false - } - } - - if isNumeric(v1) && isNumeric(v2) { - return cast.ToFloat64(v1) < cast.ToFloat64(v2) - } - - s1 := cast.ToString(v1) - s2 := cast.ToString(v2) - - return s1 < s2 - } - - pages, _ := spc.get(key, pageBy(paramsKeyComparator).Sort, p) - - return pages -} diff --git a/hugolib/pageSort_test.go b/hugolib/pageSort_test.go deleted file mode 100644 index 915947fd3..000000000 --- a/hugolib/pageSort_test.go +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestDefaultSort(t *testing.T) { - t.Parallel() - d1 := time.Now() - d2 := d1.Add(-1 * time.Hour) - d3 := d1.Add(-2 * time.Hour) - d4 := d1.Add(-3 * time.Hour) - - s := newTestSite(t) - - p := createSortTestPages(s, 4) - - // first by weight - setSortVals([4]time.Time{d1, d2, d3, d4}, [4]string{"b", "a", "c", "d"}, [4]int{4, 3, 2, 1}, p) - p.sort() - - assert.Equal(t, 1, p[0].Weight) - - // Consider zero weight, issue #2673 - setSortVals([4]time.Time{d1, d2, d3, d4}, [4]string{"b", "a", "d", "c"}, [4]int{0, 0, 0, 1}, p) - p.sort() - - assert.Equal(t, 1, p[0].Weight) - - // next by date - setSortVals([4]time.Time{d3, d4, d1, d2}, [4]string{"a", "b", "c", "d"}, [4]int{1, 1, 1, 1}, p) - p.sort() - assert.Equal(t, d1, p[0].Date) - - // finally by link title - setSortVals([4]time.Time{d3, d3, d3, d3}, [4]string{"b", "c", "a", "d"}, [4]int{1, 1, 1, 1}, p) - p.sort() - assert.Equal(t, "al", p[0].LinkTitle()) - assert.Equal(t, "bl", p[1].LinkTitle()) - assert.Equal(t, "cl", p[2].LinkTitle()) -} - -// https://github.com/gohugoio/hugo/issues/4953 -func TestSortByLinkTitle(t *testing.T) { - t.Parallel() - assert := require.New(t) - s := newTestSite(t) - pages := createSortTestPages(s, 6) - - for i, p := range pages { - if i < 5 { - p.title = fmt.Sprintf("title%d", i) - } - - if i > 2 { - p.linkTitle = fmt.Sprintf("linkTitle%d", i) - } - } - - pages.shuffle() - - bylt := pages.ByLinkTitle() - - for i, p := range bylt { - msg := fmt.Sprintf("test: %d", i) - if i < 3 { - assert.Equal(fmt.Sprintf("linkTitle%d", i+3), p.LinkTitle(), msg) - } else { - assert.Equal(fmt.Sprintf("title%d", i-3), p.LinkTitle(), msg) - } - } -} - -func TestSortByN(t *testing.T) { - t.Parallel() - s := newTestSite(t) - d1 := time.Now() - d2 := d1.Add(-2 * time.Hour) - d3 := d1.Add(-10 * time.Hour) - d4 := d1.Add(-20 * time.Hour) - - p := createSortTestPages(s, 4) - - for i, this := range []struct { - sortFunc func(p Pages) Pages - assertFunc func(p Pages) bool - }{ - {(Pages).ByWeight, func(p Pages) bool { return p[0].Weight == 1 }}, - {(Pages).ByTitle, func(p Pages) bool { return p[0].title == "ab" }}, - {(Pages).ByLinkTitle, func(p Pages) bool { return p[0].LinkTitle() == "abl" }}, - {(Pages).ByDate, func(p Pages) bool { return p[0].Date == d4 }}, - {(Pages).ByPublishDate, func(p Pages) bool { return p[0].PublishDate == d4 }}, - {(Pages).ByExpiryDate, func(p Pages) bool { return p[0].ExpiryDate == d4 }}, - {(Pages).ByLastmod, func(p Pages) bool { return p[1].Lastmod == d3 }}, - {(Pages).ByLength, func(p Pages) bool { return p[0].content() == "b_content" }}, - } { - setSortVals([4]time.Time{d1, d2, d3, d4}, [4]string{"b", "ab", "cde", "fg"}, [4]int{0, 3, 2, 1}, p) - - sorted := this.sortFunc(p) - if !this.assertFunc(sorted) { - t.Errorf("[%d] sort error", i) - } - } - -} - -func TestLimit(t *testing.T) { - t.Parallel() - s := newTestSite(t) - p := createSortTestPages(s, 10) - firstFive := p.Limit(5) - assert.Equal(t, 5, len(firstFive)) - for i := 0; i < 5; i++ { - assert.Equal(t, p[i], firstFive[i]) - } - assert.Equal(t, p, p.Limit(10)) - assert.Equal(t, p, p.Limit(11)) -} - -func TestPageSortReverse(t *testing.T) { - t.Parallel() - s := newTestSite(t) - p1 := createSortTestPages(s, 10) - assert.Equal(t, 0, p1[0].fuzzyWordCount) - assert.Equal(t, 9, p1[9].fuzzyWordCount) - p2 := p1.Reverse() - assert.Equal(t, 9, p2[0].fuzzyWordCount) - assert.Equal(t, 0, p2[9].fuzzyWordCount) - // cached - assert.True(t, pagesEqual(p2, p1.Reverse())) -} - -func TestPageSortByParam(t *testing.T) { - t.Parallel() - var k interface{} = "arbitrarily.nested" - s := newTestSite(t) - - unsorted := createSortTestPages(s, 10) - delete(unsorted[9].params, "arbitrarily") - - firstSetValue, _ := unsorted[0].Param(k) - secondSetValue, _ := unsorted[1].Param(k) - lastSetValue, _ := unsorted[8].Param(k) - unsetValue, _ := unsorted[9].Param(k) - - assert.Equal(t, "xyz100", firstSetValue) - assert.Equal(t, "xyz99", secondSetValue) - assert.Equal(t, "xyz92", lastSetValue) - assert.Equal(t, nil, unsetValue) - - sorted := unsorted.ByParam("arbitrarily.nested") - firstSetSortedValue, _ := sorted[0].Param(k) - secondSetSortedValue, _ := sorted[1].Param(k) - lastSetSortedValue, _ := sorted[8].Param(k) - unsetSortedValue, _ := sorted[9].Param(k) - - assert.Equal(t, firstSetValue, firstSetSortedValue) - assert.Equal(t, secondSetValue, lastSetSortedValue) - assert.Equal(t, lastSetValue, secondSetSortedValue) - assert.Equal(t, unsetValue, unsetSortedValue) -} - -func TestPageSortByParamNumeric(t *testing.T) { - t.Parallel() - var k interface{} = "arbitrarily.nested" - s := newTestSite(t) - - n := 10 - unsorted := createSortTestPages(s, n) - for i := 0; i < n; i++ { - v := 100 - i - if i%2 == 0 { - v = 100.0 - i - } - - unsorted[i].params = map[string]interface{}{ - "arbitrarily": map[string]interface{}{ - "nested": v, - }, - } - } - delete(unsorted[9].params, "arbitrarily") - - firstSetValue, _ := unsorted[0].Param(k) - secondSetValue, _ := unsorted[1].Param(k) - lastSetValue, _ := unsorted[8].Param(k) - unsetValue, _ := unsorted[9].Param(k) - - assert.Equal(t, 100, firstSetValue) - assert.Equal(t, 99, secondSetValue) - assert.Equal(t, 92, lastSetValue) - assert.Equal(t, nil, unsetValue) - - sorted := unsorted.ByParam("arbitrarily.nested") - firstSetSortedValue, _ := sorted[0].Param(k) - secondSetSortedValue, _ := sorted[1].Param(k) - lastSetSortedValue, _ := sorted[8].Param(k) - unsetSortedValue, _ := sorted[9].Param(k) - - assert.Equal(t, 92, firstSetSortedValue) - assert.Equal(t, 93, secondSetSortedValue) - assert.Equal(t, 100, lastSetSortedValue) - assert.Equal(t, unsetValue, unsetSortedValue) -} - -func BenchmarkSortByWeightAndReverse(b *testing.B) { - s := newTestSite(b) - p := createSortTestPages(s, 300) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - p = p.ByWeight().Reverse() - } -} - -func setSortVals(dates [4]time.Time, titles [4]string, weights [4]int, pages Pages) { - for i := range dates { - pages[i].Date = dates[i] - pages[i].Lastmod = dates[i] - pages[i].Weight = weights[i] - pages[i].title = titles[i] - // make sure we compare apples and ... apples ... - pages[len(dates)-1-i].linkTitle = pages[i].title + "l" - pages[len(dates)-1-i].PublishDate = dates[i] - pages[len(dates)-1-i].ExpiryDate = dates[i] - pages[len(dates)-1-i].workContent = []byte(titles[i] + "_content") - } - lastLastMod := pages[2].Lastmod - pages[2].Lastmod = pages[1].Lastmod - pages[1].Lastmod = lastLastMod - - for _, p := range pages { - p.resetContent() - } - -} - -func createSortTestPages(s *Site, num int) Pages { - pages := make(Pages, num) - - for i := 0; i < num; i++ { - p := s.newPage(filepath.FromSlash(fmt.Sprintf("/x/y/p%d.md", i))) - p.params = map[string]interface{}{ - "arbitrarily": map[string]interface{}{ - "nested": ("xyz" + fmt.Sprintf("%v", 100-i)), - }, - } - - w := 5 - - if i%2 == 0 { - w = 10 - } - p.fuzzyWordCount = i - p.Weight = w - p.Description = "initial" - - pages[i] = p - } - - return pages -} diff --git a/hugolib/page__common.go b/hugolib/page__common.go new file mode 100644 index 000000000..5bd7223cc --- /dev/null +++ b/hugolib/page__common.go @@ -0,0 +1,112 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "sync" + + "github.com/bep/gitmap" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/compare" + "github.com/gohugoio/hugo/lazy" + "github.com/gohugoio/hugo/navigation" + "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource" +) + +type pageCommon struct { + s *Site + m *pageMeta + + // Laziliy initialized dependencies. + init *lazy.Init + + // All of these represents the common parts of a page.Page + maps.Scratcher + navigation.PageMenusProvider + page.AuthorProvider + page.PageRenderProvider + page.AlternativeOutputFormatsProvider + page.ChildCareProvider + page.FileProvider + page.GetPageProvider + page.GitInfoProvider + page.InSectionPositioner + page.OutputFormatsProvider + page.PageMetaProvider + page.Positioner + page.RawContentProvider + page.RelatedKeywordsProvider + page.RefProvider + page.ShortcodeInfoProvider + page.SitesProvider + page.DeprecatedWarningPageMethods + page.TranslationsProvider + page.TreeProvider + resource.LanguageProvider + resource.ResourceDataProvider + resource.ResourceMetaProvider + resource.ResourceParamsProvider + resource.ResourceTypesProvider + resource.TranslationKeyProvider + compare.Eqer + + // Describes how paths and URLs for this page and its descendants + // should look like. + targetPathDescriptor page.TargetPathDescriptor + + layoutDescriptor output.LayoutDescriptor + layoutDescriptorInit sync.Once + + // The parsed page content. + pageContent + + // Set if feature enabled and this is in a Git repo. + gitInfo *gitmap.GitInfo + + // Positional navigation + posNextPrev *nextPrev + posNextPrevSection *nextPrev + + // Menus + pageMenus *pageMenus + + // Internal use + page.InternalDependencies + + // The children. Regular pages will have none. + pages page.Pages + pagesInit sync.Once + + // Any bundled resources + resources resource.Resources + resourcesInit sync.Once + + translations page.Pages + allTranslations page.Pages + + // Calculated an cached translation mapping key + translationKey string + translationKeyInit sync.Once + + // Will only be set for sections and regular pages. + parent *pageState + + // Will only be set for section pages and the home page. + subSections page.Pages + + // Set in fast render mode to force render a given page. + forceRender bool +} diff --git a/hugolib/page__content.go b/hugolib/page__content.go new file mode 100644 index 000000000..1b40c2ae7 --- /dev/null +++ b/hugolib/page__content.go @@ -0,0 +1,135 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + + "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/parser/pageparser" +) + +var ( + internalSummaryDividerBase = "HUGOMORE42" + internalSummaryDividerBaseBytes = []byte(internalSummaryDividerBase) + internalSummaryDividerPre = []byte("\n\n" + internalSummaryDividerBase + "\n\n") +) + +// The content related items on a Page. +type pageContent struct { + renderable bool + selfLayout string + + truncated bool + + cmap *pageContentMap + + shortcodeState *shortcodeHandler + + source rawPageContent +} + +// returns the content to be processed by Blackfriday or similar. +func (p pageContent) contentToRender(renderedShortcodes map[string]string) []byte { + source := p.source.parsed.Input() + + c := make([]byte, 0, len(source)+(len(source)/10)) + + for _, it := range p.cmap.items { + switch v := it.(type) { + case pageparser.Item: + c = append(c, source[v.Pos:v.Pos+len(v.Val)]...) + case pageContentReplacement: + c = append(c, v.val...) + case *shortcode: + if v.doMarkup || !p.renderable { + // Insert the rendered shortcode. + renderedShortcode, found := renderedShortcodes[v.placeholder] + if !found { + // This should never happen. + panic(fmt.Sprintf("rendered shortcode %q not found", v.placeholder)) + } + + c = append(c, []byte(renderedShortcode)...) + + } else { + // Insert the placeholder so we can insert the content after + // markdown processing. + c = append(c, []byte(v.placeholder)...) + + } + default: + panic(fmt.Sprintf("unkown item type %T", it)) + } + } + + return c +} + +func (p pageContent) selfLayoutForOutput(f output.Format) string { + if p.selfLayout == "" { + return "" + } + return p.selfLayout + f.Name +} + +type rawPageContent struct { + hasSummaryDivider bool + + // The AST of the parsed page. Contains information about: + // shortcodes, front matter, summary indicators. + parsed pageparser.Result + + // Returns the position in bytes after any front matter. + posMainContent int + + // These are set if we're able to determine this from the source. + posSummaryEnd int + posBodyStart int +} + +type pageContentReplacement struct { + val []byte + + source pageparser.Item +} + +type pageContentMap struct { + + // If not, we can skip any pre-rendering of shortcodes. + hasMarkdownShortcode bool + + // Indicates whether we must do placeholder replacements. + hasNonMarkdownShortcode bool + + // *shortcode, pageContentReplacement or pageparser.Item + items []interface{} +} + +func (p *pageContentMap) AddBytes(item pageparser.Item) { + p.items = append(p.items, item) +} + +func (p *pageContentMap) AddReplacement(val []byte, source pageparser.Item) { + p.items = append(p.items, pageContentReplacement{val: val, source: source}) +} + +func (p *pageContentMap) AddShortcode(s *shortcode) { + p.items = append(p.items, s) + if s.doMarkup { + p.hasMarkdownShortcode = true + } else { + p.hasNonMarkdownShortcode = true + } +} diff --git a/hugolib/page__data.go b/hugolib/page__data.go new file mode 100644 index 000000000..79a64931b --- /dev/null +++ b/hugolib/page__data.go @@ -0,0 +1,70 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "sync" + + "github.com/gohugoio/hugo/resources/page" +) + +type pageData struct { + *pageState + + dataInit sync.Once + data page.Data +} + +func (p *pageData) Data() interface{} { + p.dataInit.Do(func() { + p.data = make(page.Data) + + if p.Kind() == page.KindPage { + return + } + + switch p.Kind() { + case page.KindTaxonomy: + termInfo := p.getTaxonomyNodeInfo() + pluralInfo := termInfo.parent + + singular := pluralInfo.singular + plural := pluralInfo.plural + term := termInfo.term + taxonomy := p.s.Taxonomies[plural].Get(termInfo.termKey) + + p.data[singular] = taxonomy + p.data["Singular"] = singular + p.data["Plural"] = plural + p.data["Term"] = term + case page.KindTaxonomyTerm: + info := p.getTaxonomyNodeInfo() + plural := info.plural + singular := info.singular + + p.data["Singular"] = singular + p.data["Plural"] = plural + p.data["Terms"] = p.s.Taxonomies[plural] + // keep the following just for legacy reasons + p.data["OrderedIndex"] = p.data["Terms"] + p.data["Index"] = p.data["Terms"] + } + + // Assign the function to the map to make sure it is lazily initialized + p.data["pages"] = p.Pages + + }) + + return p.data +} diff --git a/hugolib/page__menus.go b/hugolib/page__menus.go new file mode 100644 index 000000000..0c9616a6d --- /dev/null +++ b/hugolib/page__menus.go @@ -0,0 +1,74 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "sync" + + "github.com/gohugoio/hugo/navigation" +) + +type pageMenus struct { + p *pageState + + q navigation.MenyQueryProvider + + pmInit sync.Once + pm navigation.PageMenus +} + +func (p *pageMenus) HasMenuCurrent(menuID string, me *navigation.MenuEntry) bool { + p.p.s.init.menus.Do() + p.init() + return p.q.HasMenuCurrent(menuID, me) +} + +func (p *pageMenus) IsMenuCurrent(menuID string, inme *navigation.MenuEntry) bool { + p.p.s.init.menus.Do() + p.init() + return p.q.IsMenuCurrent(menuID, inme) +} + +func (p *pageMenus) Menus() navigation.PageMenus { + // There is a reverse dependency here. initMenus will, once, build the + // site menus and update any relevant page. + p.p.s.init.menus.Do() + + return p.menus() +} + +func (p *pageMenus) menus() navigation.PageMenus { + p.init() + return p.pm + +} + +func (p *pageMenus) init() { + p.pmInit.Do(func() { + p.q = navigation.NewMenuQueryProvider( + p.p.s.Info.sectionPagesMenu, + p, + p.p.s, + p.p, + ) + + var err error + p.pm, err = navigation.PageMenusFromPage(p.p) + if err != nil { + p.p.s.Log.ERROR.Println(p.p.wrapError(err)) + } + + }) + +} diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go new file mode 100644 index 000000000..8532f5016 --- /dev/null +++ b/hugolib/page__meta.go @@ -0,0 +1,652 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "path" + "regexp" + "strings" + "time" + + "github.com/gohugoio/hugo/related" + + "github.com/gohugoio/hugo/source" + "github.com/markbates/inflect" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/helpers" + + "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/page/pagemeta" + "github.com/gohugoio/hugo/resources/resource" + "github.com/spf13/cast" +) + +var cjkRe = regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}`) + +type pageMeta struct { + // kind is the discriminator that identifies the different page types + // in the different page collections. This can, as an example, be used + // to to filter regular pages, find sections etc. + // Kind will, for the pages available to the templates, be one of: + // page, home, section, taxonomy and taxonomyTerm. + // It is of string type to make it easy to reason about in + // the templates. + kind string + + // This is a standalone page not part of any page collection. These + // include sitemap, robotsTXT and similar. It will have no pageOutputs, but + // a fixed pageOutput. + standalone bool + + bundleType string + + // Params contains configuration defined in the params section of page frontmatter. + params map[string]interface{} + + title string + linkTitle string + + resourcePath string + + weight int + + markup string + contentType string + + // whether the content is in a CJK language. + isCJKLanguage bool + + layout string + + aliases []string + + draft bool + + description string + keywords []string + + urlPaths pagemeta.URLPath + + resource.Dates + + // This is enabled if it is a leaf bundle (the "index.md" type) and it is marked as headless in front matter. + // Being headless means that + // 1. The page itself is not rendered to disk + // 2. It is not available in .Site.Pages etc. + // 3. But you can get it via .Site.GetPage + headless bool + + // A key that maps to translation(s) of this page. This value is fetched + // from the page front matter. + translationKey string + + // From front matter. + configuredOutputFormats output.Formats + + // This is the raw front matter metadata that is going to be assigned to + // the Resources above. + resourcesMetadata []map[string]interface{} + + f source.File + + sections []string + + // Sitemap overrides from front matter. + sitemap config.Sitemap + + s *Site + + renderingConfig *helpers.BlackFriday +} + +func (p *pageMeta) Aliases() []string { + return p.aliases +} + +func (p *pageMeta) Author() page.Author { + authors := p.Authors() + + for _, author := range authors { + return author + } + return page.Author{} +} + +func (p *pageMeta) Authors() page.AuthorList { + authorKeys, ok := p.params["authors"] + if !ok { + return page.AuthorList{} + } + authors := authorKeys.([]string) + if len(authors) < 1 || len(p.s.Info.Authors) < 1 { + return page.AuthorList{} + } + + al := make(page.AuthorList) + for _, author := range authors { + a, ok := p.s.Info.Authors[author] + if ok { + al[author] = a + } + } + return al +} + +func (p *pageMeta) BundleType() string { + return p.bundleType +} + +func (p *pageMeta) Description() string { + return p.description +} + +func (p *pageMeta) Lang() string { + return p.s.Lang() +} + +func (p *pageMeta) Draft() bool { + return p.draft +} + +func (p *pageMeta) File() source.File { + return p.f +} + +func (p *pageMeta) IsHome() bool { + return p.Kind() == page.KindHome +} + +func (p *pageMeta) Keywords() []string { + return p.keywords +} + +func (p *pageMeta) Kind() string { + return p.kind +} + +func (p *pageMeta) Layout() string { + return p.layout +} + +func (p *pageMeta) LinkTitle() string { + if p.linkTitle != "" { + return p.linkTitle + } + + return p.Title() +} + +func (p *pageMeta) Name() string { + if p.resourcePath != "" { + return p.resourcePath + } + return p.Title() +} + +func (p *pageMeta) IsNode() bool { + return !p.IsPage() +} + +func (p *pageMeta) IsPage() bool { + return p.Kind() == page.KindPage +} + +// Param is a convenience method to do lookups in Page's and Site's Params map, +// in that order. +// +// This method is also implemented on SiteInfo. +// TODO(bep) interface +func (p *pageMeta) Param(key interface{}) (interface{}, error) { + return resource.Param(p, p.s.Info.Params(), key) +} + +func (p *pageMeta) Params() map[string]interface{} { + return p.params +} + +func (p *pageMeta) Path() string { + if p.File() != nil { + return p.File().Path() + } + return p.SectionsPath() +} + +// RelatedKeywords implements the related.Document interface needed for fast page searches. +func (p *pageMeta) RelatedKeywords(cfg related.IndexConfig) ([]related.Keyword, error) { + + v, err := p.Param(cfg.Name) + if err != nil { + return nil, err + } + + return cfg.ToKeywords(v) +} + +func (p *pageMeta) IsSection() bool { + return p.Kind() == page.KindSection +} + +func (p *pageMeta) Section() string { + if p.IsHome() { + return "" + } + + if p.IsNode() { + if len(p.sections) == 0 { + // May be a sitemap or similar. + return "" + } + return p.sections[0] + } + + if p.File() != nil { + return p.File().Section() + } + + panic("invalid page state") + +} + +func (p *pageMeta) SectionsEntries() []string { + return p.sections +} + +func (p *pageMeta) SectionsPath() string { + return path.Join(p.SectionsEntries()...) +} + +func (p *pageMeta) Sitemap() config.Sitemap { + return p.sitemap +} + +func (p *pageMeta) Title() string { + return p.title +} + +func (p *pageMeta) Type() string { + if p.contentType != "" { + return p.contentType + } + + if x := p.Section(); x != "" { + return x + } + + return "page" +} + +func (p *pageMeta) Weight() int { + return p.weight +} + +func (pm *pageMeta) setMetadata(p *pageState, frontmatter map[string]interface{}) error { + if frontmatter == nil { + return errors.New("missing frontmatter data") + } + + pm.params = make(map[string]interface{}) + + // Needed for case insensitive fetching of params values + maps.ToLower(frontmatter) + + var mtime time.Time + if p.File().FileInfo() != nil { + mtime = p.File().FileInfo().ModTime() + } + + var gitAuthorDate time.Time + if p.gitInfo != nil { + gitAuthorDate = p.gitInfo.AuthorDate + } + + descriptor := &pagemeta.FrontMatterDescriptor{ + Frontmatter: frontmatter, + Params: pm.params, + Dates: &pm.Dates, + PageURLs: &pm.urlPaths, + BaseFilename: p.File().ContentBaseName(), + ModTime: mtime, + GitAuthorDate: gitAuthorDate, + } + + // Handle the date separately + // TODO(bep) we need to "do more" in this area so this can be split up and + // more easily tested without the Page, but the coupling is strong. + err := pm.s.frontmatterHandler.HandleDates(descriptor) + if err != nil { + p.s.Log.ERROR.Printf("Failed to handle dates for page %q: %s", p.pathOrTitle(), err) + } + + var sitemapSet bool + + var draft, published, isCJKLanguage *bool + for k, v := range frontmatter { + loki := strings.ToLower(k) + + if loki == "published" { // Intentionally undocumented + vv, err := cast.ToBoolE(v) + if err == nil { + published = &vv + } + // published may also be a date + continue + } + + if pm.s.frontmatterHandler.IsDateKey(loki) { + continue + } + + switch loki { + case "title": + pm.title = cast.ToString(v) + pm.params[loki] = pm.title + case "linktitle": + pm.linkTitle = cast.ToString(v) + pm.params[loki] = pm.linkTitle + case "description": + pm.description = cast.ToString(v) + pm.params[loki] = pm.description + case "slug": + // Don't start or end with a - + pm.urlPaths.Slug = strings.Trim(cast.ToString(v), "-") + pm.params[loki] = pm.Slug() + case "url": + if url := cast.ToString(v); strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { + return fmt.Errorf("only relative URLs are supported, %v provided", url) + } + pm.urlPaths.URL = cast.ToString(v) + pm.params[loki] = pm.urlPaths.URL + case "type": + pm.contentType = cast.ToString(v) + pm.params[loki] = pm.contentType + case "keywords": + pm.keywords = cast.ToStringSlice(v) + pm.params[loki] = pm.keywords + case "headless": + // For now, only the leaf bundles ("index.md") can be headless (i.e. produce no output). + // We may expand on this in the future, but that gets more complex pretty fast. + if p.File().TranslationBaseName() == "index" { + pm.headless = cast.ToBool(v) + } + pm.params[loki] = pm.headless + case "outputs": + o := cast.ToStringSlice(v) + if len(o) > 0 { + // Output formats are exlicitly set in front matter, use those. + outFormats, err := p.s.outputFormatsConfig.GetByNames(o...) + + if err != nil { + p.s.Log.ERROR.Printf("Failed to resolve output formats: %s", err) + } else { + pm.configuredOutputFormats = outFormats + pm.params[loki] = outFormats + } + + } + case "draft": + draft = new(bool) + *draft = cast.ToBool(v) + case "layout": + pm.layout = cast.ToString(v) + pm.params[loki] = pm.layout + case "markup": + pm.markup = cast.ToString(v) + pm.params[loki] = pm.markup + case "weight": + pm.weight = cast.ToInt(v) + pm.params[loki] = pm.weight + case "aliases": + pm.aliases = cast.ToStringSlice(v) + for _, alias := range pm.aliases { + if strings.HasPrefix(alias, "http://") || strings.HasPrefix(alias, "https://") { + return fmt.Errorf("only relative aliases are supported, %v provided", alias) + } + } + pm.params[loki] = pm.aliases + case "sitemap": + p.m.sitemap = config.DecodeSitemap(p.s.siteCfg.sitemap, cast.ToStringMap(v)) + pm.params[loki] = p.m.sitemap + sitemapSet = true + case "iscjklanguage": + isCJKLanguage = new(bool) + *isCJKLanguage = cast.ToBool(v) + case "translationkey": + pm.translationKey = cast.ToString(v) + pm.params[loki] = pm.translationKey + case "resources": + var resources []map[string]interface{} + handled := true + + switch vv := v.(type) { + case []map[interface{}]interface{}: + for _, vvv := range vv { + resources = append(resources, cast.ToStringMap(vvv)) + } + case []map[string]interface{}: + resources = append(resources, vv...) + case []interface{}: + for _, vvv := range vv { + switch vvvv := vvv.(type) { + case map[interface{}]interface{}: + resources = append(resources, cast.ToStringMap(vvvv)) + case map[string]interface{}: + resources = append(resources, vvvv) + } + } + default: + handled = false + } + + if handled { + pm.params[loki] = resources + pm.resourcesMetadata = resources + break + } + fallthrough + + default: + // If not one of the explicit values, store in Params + switch vv := v.(type) { + case bool: + pm.params[loki] = vv + case string: + pm.params[loki] = vv + case int64, int32, int16, int8, int: + pm.params[loki] = vv + case float64, float32: + pm.params[loki] = vv + case time.Time: + pm.params[loki] = vv + default: // handle array of strings as well + switch vvv := vv.(type) { + case []interface{}: + if len(vvv) > 0 { + switch vvv[0].(type) { + case map[interface{}]interface{}: // Proper parsing structured array from YAML based FrontMatter + pm.params[loki] = vvv + case map[string]interface{}: // Proper parsing structured array from JSON based FrontMatter + pm.params[loki] = vvv + case []interface{}: + pm.params[loki] = vvv + default: + a := make([]string, len(vvv)) + for i, u := range vvv { + a[i] = cast.ToString(u) + } + + pm.params[loki] = a + } + } else { + pm.params[loki] = []string{} + } + default: + pm.params[loki] = vv + } + } + } + } + + if !sitemapSet { + pm.sitemap = p.s.siteCfg.sitemap + } + + pm.markup = helpers.GuessType(pm.markup) + + if draft != nil && published != nil { + pm.draft = *draft + p.m.s.Log.WARN.Printf("page %q has both draft and published settings in its frontmatter. Using draft.", p.File().Filename()) + } else if draft != nil { + pm.draft = *draft + } else if published != nil { + pm.draft = !*published + } + pm.params["draft"] = pm.draft + + if isCJKLanguage != nil { + pm.isCJKLanguage = *isCJKLanguage + } else if p.s.siteCfg.hasCJKLanguage { + if cjkRe.Match(p.source.parsed.Input()) { + pm.isCJKLanguage = true + } else { + pm.isCJKLanguage = false + } + } + + pm.params["iscjklanguage"] = p.m.isCJKLanguage + + return nil +} + +func (p *pageMeta) applyDefaultValues() error { + if p.markup == "" { + if p.File() != nil { + // Fall back to {file extension + p.markup = helpers.GuessType(p.File().Ext()) + } + if p.markup == "" { + p.markup = "unknown" + } + } + + if p.title == "" { + switch p.Kind() { + case page.KindHome: + p.title = p.s.Info.title + case page.KindSection: + sectionName := helpers.FirstUpper(p.sections[0]) + if p.s.Cfg.GetBool("pluralizeListTitles") { + p.title = inflect.Pluralize(sectionName) + } else { + p.title = sectionName + } + case page.KindTaxonomy: + key := p.sections[len(p.sections)-1] + p.title = strings.Replace(p.s.titleFunc(key), "-", " ", -1) + case page.KindTaxonomyTerm: + p.title = p.s.titleFunc(p.sections[0]) + case kind404: + p.title = "404 Page not found" + + } + } + + if p.IsNode() { + p.bundleType = "branch" + } else { + source := p.File() + if fi, ok := source.(*fileInfo); ok { + switch fi.bundleTp { + case bundleBranch: + p.bundleType = "branch" + case bundleLeaf: + p.bundleType = "leaf" + } + } + } + + bfParam := getParamToLower(p, "blackfriday") + if bfParam != nil { + p.renderingConfig = p.s.ContentSpec.BlackFriday + + // Create a copy so we can modify it. + bf := *p.s.ContentSpec.BlackFriday + p.renderingConfig = &bf + pageParam := cast.ToStringMap(bfParam) + if err := mapstructure.Decode(pageParam, &p.renderingConfig); err != nil { + return errors.WithMessage(err, "failed to decode rendering config") + } + } + + return nil + +} + +// The output formats this page will be rendered to. +func (m *pageMeta) outputFormats() output.Formats { + if len(m.configuredOutputFormats) > 0 { + return m.configuredOutputFormats + } + + return m.s.outputFormats[m.Kind()] +} + +func (p *pageMeta) Slug() string { + return p.urlPaths.Slug +} + +func getParam(m resource.ResourceParamsProvider, key string, stringToLower bool) interface{} { + v := m.Params()[strings.ToLower(key)] + + if v == nil { + return nil + } + + switch val := v.(type) { + case bool: + return val + case string: + if stringToLower { + return strings.ToLower(val) + } + return val + case int64, int32, int16, int8, int: + return cast.ToInt(v) + case float64, float32: + return cast.ToFloat64(v) + case time.Time: + return val + case []string: + if stringToLower { + return helpers.SliceToLower(val) + } + return v + case map[string]interface{}: // JSON and TOML + return v + case map[interface{}]interface{}: // YAML + return v + } + + //p.s.Log.ERROR.Printf("GetParam(\"%s\"): Unknown type %s\n", key, reflect.TypeOf(v)) + return nil +} + +func getParamToLower(m resource.ResourceParamsProvider, key string) interface{} { + return getParam(m, key, true) +} diff --git a/hugolib/page__new.go b/hugolib/page__new.go new file mode 100644 index 000000000..0f419b5da --- /dev/null +++ b/hugolib/page__new.go @@ -0,0 +1,291 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "html/template" + "strings" + + "github.com/gohugoio/hugo/common/hugo" + + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/source" + + "github.com/gohugoio/hugo/parser/pageparser" + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/output" + + "github.com/gohugoio/hugo/lazy" + + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource" +) + +func newPageBase(metaProvider *pageMeta) (*pageState, error) { + if metaProvider.s == nil { + panic("must provide a Site") + } + + s := metaProvider.s + + ps := &pageState{ + pageOutput: nopPageOutput, + pageCommon: &pageCommon{ + FileProvider: metaProvider, + AuthorProvider: metaProvider, + Scratcher: maps.NewScratcher(), + Positioner: page.NopPage, + InSectionPositioner: page.NopPage, + ResourceMetaProvider: metaProvider, + ResourceParamsProvider: metaProvider, + PageMetaProvider: metaProvider, + RelatedKeywordsProvider: metaProvider, + OutputFormatsProvider: page.NopPage, + ResourceTypesProvider: pageTypesProvider, + RefProvider: page.NopPage, + ShortcodeInfoProvider: page.NopPage, + LanguageProvider: s, + + InternalDependencies: s, + init: lazy.New(), + m: metaProvider, + s: s}, + } + + siteAdapter := pageSiteAdapter{s: s, p: ps} + + deprecatedWarningPage := struct { + source.FileWithoutOverlap + page.DeprecatedWarningPageMethods1 + }{ + FileWithoutOverlap: metaProvider.File(), + DeprecatedWarningPageMethods1: &pageDeprecatedWarning{p: ps}, + } + + ps.DeprecatedWarningPageMethods = page.NewDeprecatedWarningPage(deprecatedWarningPage) + ps.pageMenus = &pageMenus{p: ps} + ps.PageMenusProvider = ps.pageMenus + ps.GetPageProvider = siteAdapter + ps.GitInfoProvider = ps + ps.TranslationsProvider = ps + ps.ResourceDataProvider = &pageData{pageState: ps} + ps.RawContentProvider = ps + ps.ChildCareProvider = ps + ps.TreeProvider = pageTree{p: ps} + ps.Eqer = ps + ps.TranslationKeyProvider = ps + ps.ShortcodeInfoProvider = ps + ps.PageRenderProvider = ps + ps.AlternativeOutputFormatsProvider = ps + + return ps, nil + +} + +func newPageFromMeta(metaProvider *pageMeta) (*pageState, error) { + ps, err := newPageBase(metaProvider) + if err != nil { + return nil, err + } + + if err := metaProvider.applyDefaultValues(); err != nil { + return nil, err + } + + ps.init.Add(func() (interface{}, error) { + pp, err := newPagePaths(metaProvider.s, ps, metaProvider) + if err != nil { + return nil, err + } + + makeOut := func(f output.Format, render bool) *pageOutput { + return newPageOutput(nil, ps, pp, f, render) + } + + if ps.m.standalone { + ps.pageOutput = makeOut(ps.m.outputFormats()[0], true) + } else { + ps.pageOutputs = make([]*pageOutput, len(ps.s.h.renderFormats)) + created := make(map[string]*pageOutput) + outputFormatsForPage := ps.m.outputFormats() + for i, f := range ps.s.h.renderFormats { + po, found := created[f.Name] + if !found { + _, shouldRender := outputFormatsForPage.GetByName(f.Name) + po = makeOut(f, shouldRender) + created[f.Name] = po + } + ps.pageOutputs[i] = po + } + } + + if err := ps.initCommonProviders(pp); err != nil { + return nil, err + } + + return nil, nil + + }) + + return ps, err + +} + +// Used by the legacy 404, sitemap and robots.txt rendering +func newPageStandalone(m *pageMeta, f output.Format) (*pageState, error) { + m.configuredOutputFormats = output.Formats{f} + m.standalone = true + p, err := newPageFromMeta(m) + + if err != nil { + return nil, err + } + + if err := p.initPage(); err != nil { + return nil, err + } + + return p, nil + +} + +func newPageWithContent(f *fileInfo, s *Site, content resource.OpenReadSeekCloser) (*pageState, error) { + sections := s.sectionsFromFile(f) + kind := s.kindFromFileInfoOrSections(f, sections) + if kind == page.KindTaxonomy { + s.PathSpec.MakePathsSanitized(sections) + } + + metaProvider := &pageMeta{kind: kind, sections: sections, s: s, f: f} + + ps, err := newPageBase(metaProvider) + if err != nil { + return nil, err + } + + gi, err := s.h.gitInfoForPage(ps) + if err != nil { + return nil, errors.Wrap(err, "failed to load Git data") + } + ps.gitInfo = gi + + r, err := content() + if err != nil { + return nil, err + } + defer r.Close() + + parseResult, err := pageparser.Parse( + r, + pageparser.Config{EnableEmoji: s.siteCfg.enableEmoji}, + ) + if err != nil { + return nil, err + } + + ps.pageContent = pageContent{ + source: rawPageContent{ + parsed: parseResult, + posMainContent: -1, + posSummaryEnd: -1, + posBodyStart: -1, + }, + } + + ps.shortcodeState = newShortcodeHandler(ps, ps.s, nil) + + if err := ps.mapContent(metaProvider); err != nil { + return nil, ps.wrapError(err) + } + + if err := metaProvider.applyDefaultValues(); err != nil { + return nil, err + } + + ps.init.Add(func() (interface{}, error) { + reuseContent := ps.renderable && !ps.shortcodeState.hasShortcodes() + + // Creates what's needed for each output format. + contentPerOutput := newPageContentOutput(ps) + + pp, err := newPagePaths(s, ps, metaProvider) + if err != nil { + return nil, err + } + + // Prepare output formats for all sites. + ps.pageOutputs = make([]*pageOutput, len(ps.s.h.renderFormats)) + created := make(map[string]*pageOutput) + outputFormatsForPage := ps.m.outputFormats() + + for i, f := range ps.s.h.renderFormats { + if po, found := created[f.Name]; found { + ps.pageOutputs[i] = po + continue + } + + _, render := outputFormatsForPage.GetByName(f.Name) + var contentProvider *pageContentOutput + if reuseContent && i > 0 { + contentProvider = ps.pageOutputs[0].cp + } else { + var err error + contentProvider, err = contentPerOutput(f) + if err != nil { + return nil, err + } + } + + po := newPageOutput(contentProvider, ps, pp, f, render) + ps.pageOutputs[i] = po + created[f.Name] = po + } + + if err := ps.initCommonProviders(pp); err != nil { + return nil, err + } + + return nil, nil + }) + + return ps, nil +} + +type pageDeprecatedWarning struct { + p *pageState +} + +func (p *pageDeprecatedWarning) IsDraft() bool { return p.p.m.draft } +func (p *pageDeprecatedWarning) Hugo() hugo.Info { return p.p.s.Info.Hugo() } +func (p *pageDeprecatedWarning) LanguagePrefix() string { return p.p.s.Info.LanguagePrefix } +func (p *pageDeprecatedWarning) GetParam(key string) interface{} { + return p.p.m.params[strings.ToLower(key)] +} +func (p *pageDeprecatedWarning) RSSLink() template.URL { + f := p.p.OutputFormats().Get("RSS") + if f == nil { + return "" + } + return template.URL(f.Permalink()) +} +func (p *pageDeprecatedWarning) URL() string { + if p.p.IsPage() && p.p.m.urlPaths.URL != "" { + // This is the url set in front matter + return p.p.m.urlPaths.URL + } + // Fall back to the relative permalink. + return p.p.RelPermalink() + +} diff --git a/hugolib/page__output.go b/hugolib/page__output.go new file mode 100644 index 000000000..d38d7c852 --- /dev/null +++ b/hugolib/page__output.go @@ -0,0 +1,107 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource" +) + +func newPageOutput( + cp *pageContentOutput, // may be nil + ps *pageState, + pp pagePaths, + f output.Format, + render bool) *pageOutput { + + var targetPathsProvider targetPathsHolder + var linksProvider resource.ResourceLinksProvider + + ft, found := pp.targetPaths[f.Name] + if !found { + // Link to the main output format + ft = pp.targetPaths[pp.OutputFormats()[0].Format.Name] + } + targetPathsProvider = ft + linksProvider = ft + + var paginatorProvider page.PaginatorProvider = page.NopPage + var pag *pagePaginator + + if render && ps.IsNode() { + pag = &pagePaginator{source: ps} + paginatorProvider = pag + } + + var contentProvider page.ContentProvider = page.NopPage + var tableOfContentsProvider page.TableOfContentsProvider = page.NopPage + + if cp != nil { + contentProvider = cp + tableOfContentsProvider = cp + } + + providers := struct { + page.ContentProvider + page.TableOfContentsProvider + page.PaginatorProvider + resource.ResourceLinksProvider + targetPather + }{ + contentProvider, + tableOfContentsProvider, + paginatorProvider, + linksProvider, + targetPathsProvider, + } + + po := &pageOutput{ + f: f, + cp: cp, + pagePerOutputProviders: providers, + render: render, + paginator: pag, + } + + return po + +} + +// We create a pageOutput for every output format combination, even if this +// particular page isn't configured to be rendered to that format. +type pageOutput struct { + // Set if this page isn't configured to be rendered to this format. + render bool + + f output.Format + + // Only set if render is set. + // Note that this will be lazily initialized, so only used if actually + // used in template(s). + paginator *pagePaginator + + // This interface provides the functionality that is specific for this + // output format. + pagePerOutputProviders + + // This may be nil. + cp *pageContentOutput +} + +func (p *pageOutput) enablePlaceholders() { + if p.cp != nil { + p.cp.enablePlaceholders() + } +} diff --git a/hugolib/page__paginator.go b/hugolib/page__paginator.go new file mode 100644 index 000000000..93701e799 --- /dev/null +++ b/hugolib/page__paginator.go @@ -0,0 +1,83 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "sync" + + "github.com/gohugoio/hugo/resources/page" +) + +type pagePaginator struct { + paginatorInit sync.Once + current *page.Pager + + source *pageState +} + +func (p *pagePaginator) Paginate(seq interface{}, options ...interface{}) (*page.Pager, error) { + var initErr error + p.paginatorInit.Do(func() { + pagerSize, err := page.ResolvePagerSize(p.source.s.Cfg, options...) + if err != nil { + initErr = err + return + } + + pd := p.source.targetPathDescriptor + pd.Type = p.source.outputFormat() + paginator, err := page.Paginate(pd, seq, pagerSize) + if err != nil { + initErr = err + return + } + + p.current = paginator.Pagers()[0] + + }) + + if initErr != nil { + return nil, initErr + } + + return p.current, nil +} + +func (p *pagePaginator) Paginator(options ...interface{}) (*page.Pager, error) { + var initErr error + p.paginatorInit.Do(func() { + pagerSize, err := page.ResolvePagerSize(p.source.s.Cfg, options...) + if err != nil { + initErr = err + return + } + + pd := p.source.targetPathDescriptor + pd.Type = p.source.outputFormat() + paginator, err := page.Paginate(pd, p.source.Pages(), pagerSize) + if err != nil { + initErr = err + return + } + + p.current = paginator.Pagers()[0] + + }) + + if initErr != nil { + return nil, initErr + } + + return p.current, nil +} diff --git a/hugolib/page__paths.go b/hugolib/page__paths.go new file mode 100644 index 000000000..0a5dad5ef --- /dev/null +++ b/hugolib/page__paths.go @@ -0,0 +1,148 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "net/url" + + "github.com/gohugoio/hugo/resources/page" +) + +func newPagePaths( + s *Site, + p page.Page, + pm *pageMeta) (pagePaths, error) { + + targetPathDescriptor, err := createTargetPathDescriptor(s, p, pm) + if err != nil { + return pagePaths{}, err + } + + outputFormats := pm.outputFormats() + if len(outputFormats) == 0 { + outputFormats = pm.s.outputFormats[pm.Kind()] + } + + if len(outputFormats) == 0 { + return pagePaths{}, nil + } + + if pm.headless { + outputFormats = outputFormats[:1] + } + + pageOutputFormats := make(page.OutputFormats, len(outputFormats)) + targets := make(map[string]targetPathsHolder) + + for i, f := range outputFormats { + desc := targetPathDescriptor + desc.Type = f + paths := page.CreateTargetPaths(desc) + + var relPermalink, permalink string + + if !pm.headless { + relPermalink = paths.RelPermalink(s.PathSpec) + permalink = paths.PermalinkForOutputFormat(s.PathSpec, f) + } + + pageOutputFormats[i] = page.NewOutputFormat(relPermalink, permalink, len(outputFormats) == 1, f) + + // Use the main format for permalinks, usually HTML. + permalinksIndex := 0 + if f.Permalinkable { + // Unless it's permalinkable + permalinksIndex = i + } + + targets[f.Name] = targetPathsHolder{ + paths: paths, + OutputFormat: pageOutputFormats[permalinksIndex]} + + } + + return pagePaths{ + outputFormats: pageOutputFormats, + targetPaths: targets, + targetPathDescriptor: targetPathDescriptor, + }, nil + +} + +type pagePaths struct { + outputFormats page.OutputFormats + + targetPaths map[string]targetPathsHolder + targetPathDescriptor page.TargetPathDescriptor +} + +func (l pagePaths) OutputFormats() page.OutputFormats { + return l.outputFormats +} + +func createTargetPathDescriptor(s *Site, p page.Page, pm *pageMeta) (page.TargetPathDescriptor, error) { + var ( + dir string + baseName string + ) + + d := s.Deps + + if p.File() != nil { + dir = p.File().Dir() + baseName = p.File().TranslationBaseName() + } + + alwaysInSubDir := p.Kind() == kindSitemap + + desc := page.TargetPathDescriptor{ + PathSpec: d.PathSpec, + Kind: p.Kind(), + Sections: p.SectionsEntries(), + UglyURLs: s.Info.uglyURLs(p), + ForcePrefix: s.h.IsMultihost() || alwaysInSubDir, + Dir: dir, + URL: pm.urlPaths.URL, + } + + if pm.Slug() != "" { + desc.BaseName = pm.Slug() + } else { + desc.BaseName = baseName + } + + desc.PrefixFilePath = s.getLanguageTargetPathLang(alwaysInSubDir) + desc.PrefixLink = s.getLanguagePermalinkLang(alwaysInSubDir) + + // Expand only page.KindPage and page.KindTaxonomy; don't expand other Kinds of Pages + // like page.KindSection or page.KindTaxonomyTerm because they are "shallower" and + // the permalink configuration values are likely to be redundant, e.g. + // naively expanding /category/:slug/ would give /category/categories/ for + // the "categories" page.KindTaxonomyTerm. + if p.Kind() == page.KindPage || p.Kind() == page.KindTaxonomy { + opath, err := d.ResourceSpec.Permalinks.Expand(p.Section(), p) + if err != nil { + return desc, err + } + + if opath != "" { + opath, _ = url.QueryUnescape(opath) + desc.ExpandedPermalink = opath + } + + } + + return desc, nil + +} diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go new file mode 100644 index 000000000..05b35cc87 --- /dev/null +++ b/hugolib/page__per_output.go @@ -0,0 +1,445 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "bytes" + "context" + "fmt" + "html/template" + "strings" + "sync" + "unicode/utf8" + + "github.com/gohugoio/hugo/lazy" + + bp "github.com/gohugoio/hugo/bufferpool" + "github.com/gohugoio/hugo/tpl" + + "github.com/gohugoio/hugo/output" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource" +) + +var ( + nopTargetPath = targetPathsHolder{} + nopPagePerOutput = struct { + resource.ResourceLinksProvider + page.ContentProvider + page.PageRenderProvider + page.PaginatorProvider + page.TableOfContentsProvider + page.AlternativeOutputFormatsProvider + + targetPather + }{ + page.NopPage, + page.NopPage, + page.NopPage, + page.NopPage, + page.NopPage, + page.NopPage, + nopTargetPath, + } +) + +func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutput, error) { + + parent := p.init + + return func(f output.Format) (*pageContentOutput, error) { + cp := &pageContentOutput{ + p: p, + f: f, + } + + initContent := func() error { + var err error + var hasVariants bool + + cp.contentPlaceholders, hasVariants, err = p.shortcodeState.renderShortcodesForPage(p, f) + if err != nil { + return err + } + + if p.render && !hasVariants { + // We can reuse this for the other output formats + cp.enableReuse() + } + + cp.workContent = p.contentToRender(cp.contentPlaceholders) + + isHTML := cp.p.m.markup == "html" + + if p.renderable { + if !isHTML { + cp.workContent = cp.renderContent(p, cp.workContent) + tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent) + cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents) + cp.workContent = tmpContent + } + + if cp.placeholdersEnabled { + // ToC was accessed via .Page.TableOfContents in the shortcode, + // at a time when the ToC wasn't ready. + cp.contentPlaceholders[tocShortcodePlaceholder] = string(cp.tableOfContents) + } + + if p.cmap.hasNonMarkdownShortcode || cp.placeholdersEnabled { + // There are one or more replacement tokens to be replaced. + cp.workContent, err = replaceShortcodeTokens(cp.workContent, cp.contentPlaceholders) + if err != nil { + return err + } + } + + if cp.p.source.hasSummaryDivider { + if isHTML { + src := p.source.parsed.Input() + + // Use the summary sections as they are provided by the user. + if p.source.posSummaryEnd != -1 { + cp.summary = helpers.BytesToHTML(src[p.source.posMainContent:p.source.posSummaryEnd]) + } + + if cp.p.source.posBodyStart != -1 { + cp.workContent = src[cp.p.source.posBodyStart:] + } + + } else { + summary, content, err := splitUserDefinedSummaryAndContent(cp.p.m.markup, cp.workContent) + if err != nil { + cp.p.s.Log.ERROR.Printf("Failed to set user defined summary for page %q: %s", cp.p.pathOrTitle(), err) + } else { + cp.workContent = content + cp.summary = helpers.BytesToHTML(summary) + } + } + } + } + + cp.content = helpers.BytesToHTML(cp.workContent) + + if !p.renderable { + err := cp.addSelfTemplate() + return err + } + + return nil + + } + + // Recursive loops can only happen in content files with template code (shortcodes etc.) + // Avoid creating new goroutines if we don't have to. + needTimeout := !p.renderable || p.shortcodeState.hasShortcodes() + + if needTimeout { + cp.initMain = parent.BranchdWithTimeout(p.s.siteCfg.timeout, func(ctx context.Context) (interface{}, error) { + return nil, initContent() + }) + } else { + cp.initMain = parent.Branch(func() (interface{}, error) { + return nil, initContent() + }) + } + + cp.initPlain = cp.initMain.Branch(func() (interface{}, error) { + cp.plain = helpers.StripHTML(string(cp.content)) + cp.plainWords = strings.Fields(cp.plain) + cp.setWordCounts(p.m.isCJKLanguage) + + if err := cp.setAutoSummary(); err != nil { + return err, nil + } + + return nil, nil + }) + + return cp, nil + + } + +} + +// pageContentOutput represents the Page content for a given output format. +type pageContentOutput struct { + f output.Format + + // If we can safely reuse this for other output formats. + reuse bool + reuseInit sync.Once + + p *pageState + + // Lazy load dependencies + initMain *lazy.Init + initPlain *lazy.Init + + placeholdersEnabled bool + placeholdersEnabledInit sync.Once + + // Content state + + workContent []byte + + // Temporary storage of placeholders mapped to their content. + // These are shortcodes etc. Some of these will need to be replaced + // after any markup is rendered, so they share a common prefix. + contentPlaceholders map[string]string + + // Content sections + content template.HTML + summary template.HTML + tableOfContents template.HTML + + truncated bool + + plainWords []string + plain string + fuzzyWordCount int + wordCount int + readingTime int +} + +func (p *pageContentOutput) Content() (interface{}, error) { + p.p.s.initInit(p.initMain, p.p) + return p.content, nil +} + +func (p *pageContentOutput) FuzzyWordCount() int { + p.p.s.initInit(p.initPlain, p.p) + return p.fuzzyWordCount +} + +func (p *pageContentOutput) Len() int { + p.p.s.initInit(p.initMain, p.p) + return len(p.content) +} + +func (p *pageContentOutput) Plain() string { + p.p.s.initInit(p.initPlain, p.p) + return p.plain +} + +func (p *pageContentOutput) PlainWords() []string { + p.p.s.initInit(p.initPlain, p.p) + return p.plainWords +} + +func (p *pageContentOutput) ReadingTime() int { + p.p.s.initInit(p.initPlain, p.p) + return p.readingTime +} + +func (p *pageContentOutput) Summary() template.HTML { + p.p.s.initInit(p.initMain, p.p) + if !p.p.source.hasSummaryDivider { + p.p.s.initInit(p.initPlain, p.p) + } + return p.summary +} + +func (p *pageContentOutput) TableOfContents() template.HTML { + p.p.s.initInit(p.initMain, p.p) + return p.tableOfContents +} + +func (p *pageContentOutput) Truncated() bool { + if p.p.truncated { + return true + } + p.p.s.initInit(p.initPlain, p.p) + return p.truncated +} + +func (p *pageContentOutput) WordCount() int { + p.p.s.initInit(p.initPlain, p.p) + return p.wordCount +} + +func (p *pageContentOutput) setAutoSummary() error { + if p.p.source.hasSummaryDivider { + return nil + } + + var summary string + var truncated bool + + if p.p.m.isCJKLanguage { + summary, truncated = p.p.s.ContentSpec.TruncateWordsByRune(p.plainWords) + } else { + summary, truncated = p.p.s.ContentSpec.TruncateWordsToWholeSentence(p.plain) + } + p.summary = template.HTML(summary) + + p.truncated = truncated + + return nil + +} + +func (cp *pageContentOutput) renderContent(p page.Page, content []byte) []byte { + return cp.p.s.ContentSpec.RenderBytes(&helpers.RenderingContext{ + Content: content, RenderTOC: true, PageFmt: cp.p.m.markup, + Cfg: p.Language(), + DocumentID: p.File().UniqueID(), DocumentName: p.File().Path(), + Config: cp.p.getRenderingConfig()}) +} + +func (p *pageContentOutput) setWordCounts(isCJKLanguage bool) { + if isCJKLanguage { + p.wordCount = 0 + for _, word := range p.plainWords { + runeCount := utf8.RuneCountInString(word) + if len(word) == runeCount { + p.wordCount++ + } else { + p.wordCount += runeCount + } + } + } else { + p.wordCount = helpers.TotalWords(p.plain) + } + + // TODO(bep) is set in a test. Fix that. + if p.fuzzyWordCount == 0 { + p.fuzzyWordCount = (p.wordCount + 100) / 100 * 100 + } + + if isCJKLanguage { + p.readingTime = (p.wordCount + 500) / 501 + } else { + p.readingTime = (p.wordCount + 212) / 213 + } +} + +func (p *pageContentOutput) addSelfTemplate() error { + self := p.p.selfLayoutForOutput(p.f) + err := p.p.s.TemplateHandler().AddLateTemplate(self, string(p.content)) + if err != nil { + return err + } + return nil +} + +// A callback to signal that we have inserted a placeholder into the rendered +// content. This avoids doing extra replacement work. +func (p *pageContentOutput) enablePlaceholders() { + p.placeholdersEnabledInit.Do(func() { + p.placeholdersEnabled = true + }) +} + +func (p *pageContentOutput) enableReuse() { + p.reuseInit.Do(func() { + p.reuse = true + }) +} + +// these will be shifted out when rendering a given output format. +type pagePerOutputProviders interface { + targetPather + page.ContentProvider + page.PaginatorProvider + page.TableOfContentsProvider + resource.ResourceLinksProvider +} + +type targetPather interface { + targetPaths() page.TargetPaths +} + +type targetPathsHolder struct { + paths page.TargetPaths + page.OutputFormat +} + +func (t targetPathsHolder) targetPaths() page.TargetPaths { + return t.paths +} + +func executeToString(templ tpl.Template, data interface{}) (string, error) { + b := bp.GetBuffer() + defer bp.PutBuffer(b) + if err := templ.Execute(b, data); err != nil { + return "", err + } + return b.String(), nil + +} + +func splitUserDefinedSummaryAndContent(markup string, c []byte) (summary []byte, content []byte, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("summary split failed: %s", r) + } + }() + + startDivider := bytes.Index(c, internalSummaryDividerBaseBytes) + + if startDivider == -1 { + return + } + + startTag := "p" + switch markup { + case "asciidoc": + startTag = "div" + + } + + // Walk back and forward to the surrounding tags. + start := bytes.LastIndex(c[:startDivider], []byte("<"+startTag)) + end := bytes.Index(c[startDivider:], []byte("</"+startTag)) + + if start == -1 { + start = startDivider + } else { + start = startDivider - (startDivider - start) + } + + if end == -1 { + end = startDivider + len(internalSummaryDividerBase) + } else { + end = startDivider + end + len(startTag) + 3 + } + + var addDiv bool + + switch markup { + case "rst": + addDiv = true + } + + withoutDivider := append(c[:start], bytes.Trim(c[end:], "\n")...) + + if len(withoutDivider) > 0 { + summary = bytes.TrimSpace(withoutDivider[:start]) + } + + if addDiv { + // For the rst + summary = append(append([]byte(nil), summary...), []byte("</div>")...) + } + + if err != nil { + return + } + + content = bytes.TrimSpace(withoutDivider) + + return +} diff --git a/hugolib/page__position.go b/hugolib/page__position.go new file mode 100644 index 000000000..458b3e423 --- /dev/null +++ b/hugolib/page__position.go @@ -0,0 +1,76 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "github.com/gohugoio/hugo/lazy" + "github.com/gohugoio/hugo/resources/page" +) + +func newPagePosition(n *nextPrev) pagePosition { + return pagePosition{nextPrev: n} +} + +func newPagePositionInSection(n *nextPrev) pagePositionInSection { + return pagePositionInSection{nextPrev: n} + +} + +type nextPrev struct { + init *lazy.Init + prevPage page.Page + nextPage page.Page +} + +func (n *nextPrev) next() page.Page { + n.init.Do() + return n.nextPage +} + +func (n *nextPrev) prev() page.Page { + n.init.Do() + return n.prevPage +} + +type pagePosition struct { + *nextPrev +} + +func (p pagePosition) Next() page.Page { + return p.next() +} + +func (p pagePosition) NextPage() page.Page { + return p.Next() +} + +func (p pagePosition) Prev() page.Page { + return p.prev() +} + +func (p pagePosition) PrevPage() page.Page { + return p.Prev() +} + +type pagePositionInSection struct { + *nextPrev +} + +func (p pagePositionInSection) NextInSection() page.Page { + return p.next() +} + +func (p pagePositionInSection) PrevInSection() page.Page { + return p.prev() +} diff --git a/hugolib/page_ref.go b/hugolib/page__ref.go index af1ec3e70..41bd527db 100644 --- a/hugolib/page_ref.go +++ b/hugolib/page__ref.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,24 +22,43 @@ import ( "github.com/pkg/errors" ) -type refArgs struct { - Path string - Lang string - OutputFormat string +func newPageRef(p *pageState) pageRef { + return pageRef{p: p} } -func (p *Page) decodeRefArgs(args map[string]interface{}) (refArgs, *Site, error) { +type pageRef struct { + p *pageState +} + +func (p pageRef) Ref(argsm map[string]interface{}) (string, error) { + return p.ref(argsm, p.p) +} + +func (p pageRef) RefFrom(argsm map[string]interface{}, source interface{}) (string, error) { + return p.ref(argsm, source) +} + +func (p pageRef) RelRef(argsm map[string]interface{}) (string, error) { + return p.relRef(argsm, p.p) +} + +func (p pageRef) RelRefFrom(argsm map[string]interface{}, source interface{}) (string, error) { + return p.relRef(argsm, source) +} + +func (p pageRef) decodeRefArgs(args map[string]interface{}) (refArgs, *Site, error) { var ra refArgs err := mapstructure.WeakDecode(args, &ra) if err != nil { return ra, nil, nil } - s := p.s - if ra.Lang != "" && ra.Lang != p.Lang() { + s := p.p.s + + if ra.Lang != "" && ra.Lang != p.p.s.Language().Lang { // Find correct site found := false - for _, ss := range p.s.owner.Sites { + for _, ss := range p.p.s.h.Sites { if ss.Lang() == ra.Lang { found = true s = ss @@ -47,7 +66,7 @@ func (p *Page) decodeRefArgs(args map[string]interface{}) (refArgs, *Site, error } if !found { - p.s.siteRefLinker.logNotFound(ra.Path, fmt.Sprintf("no site found with lang %q", ra.Lang), p, text.Position{}) + p.p.s.siteRefLinker.logNotFound(ra.Path, fmt.Sprintf("no site found with lang %q", ra.Lang), nil, text.Position{}) return ra, nil, nil } } @@ -55,18 +74,14 @@ func (p *Page) decodeRefArgs(args map[string]interface{}) (refArgs, *Site, error return ra, s, nil } -func (p *Page) Ref(argsm map[string]interface{}) (string, error) { - return p.ref(argsm, p) -} - -func (p *Page) ref(argsm map[string]interface{}, source interface{}) (string, error) { +func (p pageRef) ref(argsm map[string]interface{}, source interface{}) (string, error) { args, s, err := p.decodeRefArgs(argsm) if err != nil { return "", errors.Wrap(err, "invalid arguments to Ref") } if s == nil { - return p.s.siteRefLinker.notFoundURL, nil + return p.p.s.siteRefLinker.notFoundURL, nil } if args.Path == "" { @@ -77,18 +92,14 @@ func (p *Page) ref(argsm map[string]interface{}, source interface{}) (string, er } -func (p *Page) RelRef(argsm map[string]interface{}) (string, error) { - return p.relRef(argsm, p) -} - -func (p *Page) relRef(argsm map[string]interface{}, source interface{}) (string, error) { +func (p pageRef) relRef(argsm map[string]interface{}, source interface{}) (string, error) { args, s, err := p.decodeRefArgs(argsm) if err != nil { return "", errors.Wrap(err, "invalid arguments to Ref") } if s == nil { - return p.s.siteRefLinker.notFoundURL, nil + return p.p.s.siteRefLinker.notFoundURL, nil } if args.Path == "" { @@ -98,3 +109,9 @@ func (p *Page) relRef(argsm map[string]interface{}, source interface{}) (string, return s.refLink(args.Path, source, true, args.OutputFormat) } + +type refArgs struct { + Path string + Lang string + OutputFormat string +} diff --git a/hugolib/page__tree.go b/hugolib/page__tree.go new file mode 100644 index 000000000..a99e6f16c --- /dev/null +++ b/hugolib/page__tree.go @@ -0,0 +1,113 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/resources/page" +) + +type pageTree struct { + p *pageState +} + +func (pt pageTree) IsAncestor(other interface{}) (bool, error) { + if pt.p == nil { + return false, nil + } + + pp, err := unwrapPage(other) + if err != nil || pp == nil { + return false, err + } + + if pt.p.Kind() == page.KindPage && len(pt.p.SectionsEntries()) == len(pp.SectionsEntries()) { + // A regular page is never its section's ancestor. + return false, nil + } + + return helpers.HasStringsPrefix(pp.SectionsEntries(), pt.p.SectionsEntries()), nil +} + +func (pt pageTree) CurrentSection() page.Page { + p := pt.p + + if p.IsHome() || p.IsSection() { + return p + } + + return p.Parent() +} + +func (pt pageTree) IsDescendant(other interface{}) (bool, error) { + if pt.p == nil { + return false, nil + } + pp, err := unwrapPage(other) + if err != nil || pp == nil { + return false, err + } + + if pp.Kind() == page.KindPage && len(pt.p.SectionsEntries()) == len(pp.SectionsEntries()) { + // A regular page is never its section's descendant. + return false, nil + } + return helpers.HasStringsPrefix(pt.p.SectionsEntries(), pp.SectionsEntries()), nil +} + +func (pt pageTree) FirstSection() page.Page { + p := pt.p + + parent := p.Parent() + + if types.IsNil(parent) || parent.IsHome() { + return p + } + + for { + current := parent + parent = parent.Parent() + if types.IsNil(parent) || parent.IsHome() { + return current + } + } + +} + +func (pt pageTree) InSection(other interface{}) (bool, error) { + if pt.p == nil || types.IsNil(other) { + return false, nil + } + + pp, err := unwrapPage(other) + if err != nil { + return false, err + } + + if pp == nil { + return false, nil + } + + return pp.CurrentSection().Eq(pt.p.CurrentSection()), nil + +} + +func (pt pageTree) Parent() page.Page { + return pt.p.parent +} + +func (pt pageTree) Sections() page.Pages { + return pt.p.subSections +} diff --git a/hugolib/page_content.go b/hugolib/page_content.go deleted file mode 100644 index 924400aea..000000000 --- a/hugolib/page_content.go +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "bytes" - "io" - - "github.com/gohugoio/hugo/helpers" - - errors "github.com/pkg/errors" - - bp "github.com/gohugoio/hugo/bufferpool" - - "github.com/gohugoio/hugo/common/herrors" - "github.com/gohugoio/hugo/common/text" - "github.com/gohugoio/hugo/parser/metadecoders" - "github.com/gohugoio/hugo/parser/pageparser" -) - -var ( - internalSummaryDividerBase = "HUGOMORE42" - internalSummaryDividerBaseBytes = []byte(internalSummaryDividerBase) - internalSummaryDividerPre = []byte("\n\n" + internalSummaryDividerBase + "\n\n") -) - -// The content related items on a Page. -type pageContent struct { - renderable bool - - // workContent is a copy of rawContent that may be mutated during site build. - workContent []byte - - shortcodeState *shortcodeHandler - - source rawPageContent -} - -type rawPageContent struct { - hasSummaryDivider bool - - // The AST of the parsed page. Contains information about: - // shortcodes, front matter, summary indicators. - parsed pageparser.Result - - // Returns the position in bytes after any front matter. - posMainContent int -} - -// TODO(bep) lazy consolidate -func (p *Page) mapContent() error { - p.shortcodeState = newShortcodeHandler(p) - s := p.shortcodeState - p.renderable = true - p.source.posMainContent = -1 - - result := bp.GetBuffer() - defer bp.PutBuffer(result) - - iter := p.source.parsed.Iterator() - - fail := func(err error, i pageparser.Item) error { - return p.parseError(err, iter.Input(), i.Pos) - } - - // the parser is guaranteed to return items in proper order or fail, so … - // … it's safe to keep some "global" state - var currShortcode shortcode - var ordinal int - -Loop: - for { - it := iter.Next() - - switch { - case it.Type == pageparser.TypeIgnore: - case it.Type == pageparser.TypeHTMLStart: - // This is HTML without front matter. It can still have shortcodes. - p.renderable = false - result.Write(it.Val) - case it.IsFrontMatter(): - f := metadecoders.FormatFromFrontMatterType(it.Type) - m, err := metadecoders.Default.UnmarshalToMap(it.Val, f) - if err != nil { - if fe, ok := err.(herrors.FileError); ok { - return herrors.ToFileErrorWithOffset(fe, iter.LineNumber()-1) - } else { - return err - } - } - if err := p.updateMetaData(m); err != nil { - return err - } - - next := iter.Peek() - if !next.IsDone() { - p.source.posMainContent = next.Pos - } - - if !p.shouldBuild() { - // Nothing more to do. - return nil - } - - case it.Type == pageparser.TypeLeadSummaryDivider: - result.Write(internalSummaryDividerPre) - p.source.hasSummaryDivider = true - // Need to determine if the page is truncated. - f := func(item pageparser.Item) bool { - if item.IsNonWhitespace() { - p.truncated = true - - // Done - return false - } - return true - } - iter.PeekWalk(f) - - // Handle shortcode - case it.IsLeftShortcodeDelim(): - // let extractShortcode handle left delim (will do so recursively) - iter.Backup() - - currShortcode, err := s.extractShortcode(ordinal, iter, p) - - if currShortcode.name != "" { - s.nameSet[currShortcode.name] = true - } - - if err != nil { - return fail(errors.Wrap(err, "failed to extract shortcode"), it) - } - - if currShortcode.params == nil { - currShortcode.params = make([]string, 0) - } - - placeHolder := s.createShortcodePlaceholder() - result.WriteString(placeHolder) - ordinal++ - s.shortcodes.Add(placeHolder, currShortcode) - case it.Type == pageparser.TypeEmoji: - if emoji := helpers.Emoji(it.ValStr()); emoji != nil { - result.Write(emoji) - } else { - result.Write(it.Val) - } - case it.IsEOF(): - break Loop - case it.IsError(): - err := fail(errors.WithStack(errors.New(it.ValStr())), it) - currShortcode.err = err - return err - - default: - result.Write(it.Val) - } - } - - resultBytes := make([]byte, result.Len()) - copy(resultBytes, result.Bytes()) - p.workContent = resultBytes - - return nil -} - -func (p *Page) parse(reader io.Reader) error { - - parseResult, err := pageparser.Parse( - reader, - pageparser.Config{EnableEmoji: p.s.Cfg.GetBool("enableEmoji")}, - ) - if err != nil { - return err - } - - p.source = rawPageContent{ - parsed: parseResult, - } - - p.lang = p.File.Lang() - - if p.s != nil && p.s.owner != nil { - gi, enabled := p.s.owner.gitInfo.forPage(p) - if gi != nil { - p.GitInfo = gi - } else if enabled { - p.s.Log.INFO.Printf("Failed to find GitInfo for page %q", p.Path()) - } - } - - return nil -} - -func (p *Page) parseError(err error, input []byte, offset int) error { - if herrors.UnwrapFileError(err) != nil { - // Use the most specific location. - return err - } - pos := p.posFromInput(input, offset) - return herrors.NewFileError("md", -1, pos.LineNumber, pos.ColumnNumber, err) - -} - -func (p *Page) posFromInput(input []byte, offset int) text.Position { - lf := []byte("\n") - input = input[:offset] - lineNumber := bytes.Count(input, lf) + 1 - endOfLastLine := bytes.LastIndex(input, lf) - - return text.Position{ - Filename: p.pathOrTitle(), - LineNumber: lineNumber, - ColumnNumber: offset - endOfLastLine, - Offset: offset, - } -} - -func (p *Page) posFromPage(offset int) text.Position { - return p.posFromInput(p.source.parsed.Input(), offset) -} diff --git a/hugolib/page_errors.go b/hugolib/page_errors.go deleted file mode 100644 index 42e2a8835..000000000 --- a/hugolib/page_errors.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - - "github.com/gohugoio/hugo/common/herrors" - errors "github.com/pkg/errors" -) - -func (p *Page) errorf(err error, format string, a ...interface{}) error { - if herrors.UnwrapErrorWithFileContext(err) != nil { - // More isn't always better. - return err - } - args := append([]interface{}{p.Lang(), p.pathOrTitle()}, a...) - format = "[%s] page %q: " + format - if err == nil { - errors.Errorf(format, args...) - return fmt.Errorf(format, args...) - } - return errors.Wrapf(err, format, args...) -} - -func (p *Page) errWithFileContext(err error) error { - - err, _ = herrors.WithFileContextForFile( - err, - p.Filename(), - p.Filename(), - p.s.SourceSpec.Fs.Source, - herrors.SimpleLineMatcher) - - return err -} diff --git a/hugolib/page_kinds.go b/hugolib/page_kinds.go new file mode 100644 index 000000000..39de31a16 --- /dev/null +++ b/hugolib/page_kinds.go @@ -0,0 +1,40 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "github.com/gohugoio/hugo/resources/page" +) + +var ( + + // This is all the kinds we can expect to find in .Site.Pages. + allKindsInPages = []string{page.KindPage, page.KindHome, page.KindSection, page.KindTaxonomy, page.KindTaxonomyTerm} + allKinds = append(allKindsInPages, []string{kindRSS, kindSitemap, kindRobotsTXT, kind404}...) +) + +const ( + + // Temporary state. + kindUnknown = "unknown" + + // The following are (currently) temporary nodes, + // i.e. nodes we create just to render in isolation. + kindRSS = "RSS" + kindSitemap = "sitemap" + kindRobotsTXT = "robotsTXT" + kind404 = "404" + + pageResourceType = "page" +) diff --git a/hugolib/page_output.go b/hugolib/page_output.go deleted file mode 100644 index 0a3eef9a6..000000000 --- a/hugolib/page_output.go +++ /dev/null @@ -1,320 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "html/template" - "os" - "strings" - "sync" - - bp "github.com/gohugoio/hugo/bufferpool" - - "github.com/gohugoio/hugo/tpl" - - "github.com/gohugoio/hugo/resources/resource" - - "github.com/gohugoio/hugo/media" - - "github.com/gohugoio/hugo/output" -) - -// PageOutput represents one of potentially many output formats of a given -// Page. -type PageOutput struct { - *Page - - // Pagination - paginator *Pager - paginatorInit sync.Once - - // Page output specific resources - resources resource.Resources - resourcesInit sync.Once - - // Keep this to create URL/path variations, i.e. paginators. - targetPathDescriptor targetPathDescriptor - - outputFormat output.Format -} - -func (p *PageOutput) targetPath(addends ...string) (string, error) { - tp, err := p.createTargetPath(p.outputFormat, false, addends...) - if err != nil { - return "", err - } - return tp, nil -} - -func newPageOutput(p *Page, createCopy, initContent bool, f output.Format) (*PageOutput, error) { - // TODO(bep) This is only needed for tests and we should get rid of it. - if p.targetPathDescriptorPrototype == nil { - if err := p.initPaths(); err != nil { - return nil, err - } - } - - if createCopy { - p = p.copy(initContent) - } - - td, err := p.createTargetPathDescriptor(f) - - if err != nil { - return nil, err - } - - return &PageOutput{ - Page: p, - outputFormat: f, - targetPathDescriptor: td, - }, nil -} - -// copy creates a copy of this PageOutput with the lazy sync.Once vars reset -// so they will be evaluated again, for word count calculations etc. -func (p *PageOutput) copyWithFormat(f output.Format, initContent bool) (*PageOutput, error) { - c, err := newPageOutput(p.Page, true, initContent, f) - if err != nil { - return nil, err - } - c.paginator = p.paginator - return c, nil -} - -func (p *PageOutput) copy() (*PageOutput, error) { - return p.copyWithFormat(p.outputFormat, false) -} - -func (p *PageOutput) layouts(layouts ...string) ([]string, error) { - if len(layouts) == 0 && p.selfLayout != "" { - return []string{p.selfLayout}, nil - } - - layoutDescriptor := p.layoutDescriptor - - if len(layouts) > 0 { - layoutDescriptor.Layout = layouts[0] - layoutDescriptor.LayoutOverride = true - } - - return p.s.layoutHandler.For( - layoutDescriptor, - p.outputFormat) -} - -func (p *PageOutput) Render(layout ...string) template.HTML { - l, err := p.layouts(layout...) - if err != nil { - p.s.DistinctErrorLog.Printf("in .Render: Failed to resolve layout %q for page %q", layout, p.pathOrTitle()) - return "" - } - - for _, layout := range l { - templ, found := p.s.Tmpl.Lookup(layout) - if !found { - // This is legacy from when we had only one output format and - // HTML templates only. Some have references to layouts without suffix. - // We default to good old HTML. - templ, found = p.s.Tmpl.Lookup(layout + ".html") - } - if templ != nil { - res, err := executeToString(templ, p) - if err != nil { - p.s.DistinctErrorLog.Printf("in .Render: Failed to execute template %q: %s", layout, err) - return template.HTML("") - } - return template.HTML(res) - } - } - - return "" - -} - -func executeToString(templ tpl.Template, data interface{}) (string, error) { - b := bp.GetBuffer() - defer bp.PutBuffer(b) - if err := templ.Execute(b, data); err != nil { - return "", err - } - return b.String(), nil - -} - -func (p *Page) Render(layout ...string) template.HTML { - if p.mainPageOutput == nil { - panic(fmt.Sprintf("programming error: no mainPageOutput for %q", p.Path())) - } - return p.mainPageOutput.Render(layout...) -} - -// OutputFormats holds a list of the relevant output formats for a given resource. -type OutputFormats []*OutputFormat - -// OutputFormat links to a representation of a resource. -type OutputFormat struct { - // Rel constains a value that can be used to construct a rel link. - // This is value is fetched from the output format definition. - // Note that for pages with only one output format, - // this method will always return "canonical". - // As an example, the AMP output format will, by default, return "amphtml". - // - // See: - // https://www.ampproject.org/docs/guides/deploy/discovery - // - // Most other output formats will have "alternate" as value for this. - Rel string - - // It may be tempting to export this, but let us hold on to that horse for a while. - f output.Format - - p *Page -} - -// Name returns this OutputFormat's name, i.e. HTML, AMP, JSON etc. -func (o OutputFormat) Name() string { - return o.f.Name -} - -// MediaType returns this OutputFormat's MediaType (MIME type). -func (o OutputFormat) MediaType() media.Type { - return o.f.MediaType -} - -// OutputFormats gives the output formats for this Page. -func (p *Page) OutputFormats() OutputFormats { - var o OutputFormats - for _, f := range p.outputFormats { - o = append(o, newOutputFormat(p, f)) - } - return o -} - -func newOutputFormat(p *Page, f output.Format) *OutputFormat { - rel := f.Rel - isCanonical := len(p.outputFormats) == 1 - if isCanonical { - rel = "canonical" - } - return &OutputFormat{Rel: rel, f: f, p: p} -} - -// AlternativeOutputFormats gives the alternative output formats for this PageOutput. -// Note that we use the term "alternative" and not "alternate" here, as it -// does not necessarily replace the other format, it is an alternative representation. -func (p *PageOutput) AlternativeOutputFormats() (OutputFormats, error) { - var o OutputFormats - for _, of := range p.OutputFormats() { - if of.f.NotAlternative || of.f.Name == p.outputFormat.Name { - continue - } - o = append(o, of) - } - return o, nil -} - -// deleteResource removes the resource from this PageOutput and the Page. They will -// always be of the same length, but may contain different elements. -func (p *PageOutput) deleteResource(i int) { - p.resources = append(p.resources[:i], p.resources[i+1:]...) - p.Page.Resources = append(p.Page.Resources[:i], p.Page.Resources[i+1:]...) - -} - -func (p *PageOutput) Resources() resource.Resources { - p.resourcesInit.Do(func() { - // If the current out shares the same path as the main page output, we reuse - // the resource set. For the "amp" use case, we need to clone them with new - // base folder. - ff := p.outputFormats[0] - if p.outputFormat.Path == ff.Path { - p.resources = p.Page.Resources - return - } - - // Clone it with new base. - resources := make(resource.Resources, len(p.Page.Resources)) - - for i, r := range p.Page.Resources { - if c, ok := r.(resource.Cloner); ok { - // Clone the same resource with a new target. - resources[i] = c.WithNewBase(p.outputFormat.Path) - } else { - resources[i] = r - } - } - - p.resources = resources - }) - - return p.resources -} - -func (p *PageOutput) renderResources() error { - - for i, r := range p.Resources() { - src, ok := r.(resource.Source) - if !ok { - // Pages gets rendered with the owning page. - continue - } - - if err := src.Publish(); err != nil { - if os.IsNotExist(err) { - // The resource has been deleted from the file system. - // This should be extremely rare, but can happen on live reload in server - // mode when the same resource is member of different page bundles. - p.deleteResource(i) - } else { - p.s.Log.ERROR.Printf("Failed to publish Resource for page %q: %s", p.pathOrTitle(), err) - } - } else { - p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Files) - } - } - return nil -} - -// AlternativeOutputFormats is only available on the top level rendering -// entry point, and not inside range loops on the Page collections. -// This method is just here to inform users of that restriction. -func (p *Page) AlternativeOutputFormats() (OutputFormats, error) { - return nil, fmt.Errorf("AlternativeOutputFormats only available from the top level template context for page %q", p.Path()) -} - -// Get gets a OutputFormat given its name, i.e. json, html etc. -// It returns nil if not found. -func (o OutputFormats) Get(name string) *OutputFormat { - for _, f := range o { - if strings.EqualFold(f.f.Name, name) { - return f - } - } - return nil -} - -// Permalink returns the absolute permalink to this output format. -func (o *OutputFormat) Permalink() string { - rel := o.p.createRelativePermalinkForOutputFormat(o.f) - perm, _ := o.p.s.permalinkForOutputFormat(rel, o.f) - return perm -} - -// RelPermalink returns the relative permalink to this output format. -func (o *OutputFormat) RelPermalink() string { - rel := o.p.createRelativePermalinkForOutputFormat(o.f) - return o.p.s.PathSpec.PrependBasePath(rel, false) -} diff --git a/hugolib/page_paths.go b/hugolib/page_paths.go deleted file mode 100644 index 9de7b0764..000000000 --- a/hugolib/page_paths.go +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "path/filepath" - - "net/url" - "strings" - - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/output" -) - -// targetPathDescriptor describes how a file path for a given resource -// should look like on the file system. The same descriptor is then later used to -// create both the permalinks and the relative links, paginator URLs etc. -// -// The big motivating behind this is to have only one source of truth for URLs, -// and by that also get rid of most of the fragile string parsing/encoding etc. -// -// Page.createTargetPathDescriptor is the Page adapter. -// -type targetPathDescriptor struct { - PathSpec *helpers.PathSpec - - Type output.Format - Kind string - - Sections []string - - // For regular content pages this is either - // 1) the Slug, if set, - // 2) the file base name (TranslationBaseName). - BaseName string - - // Source directory. - Dir string - - // Language prefix, set if multilingual and if page should be placed in its - // language subdir. - LangPrefix string - - // Whether this is a multihost multilingual setup. - IsMultihost bool - - // URL from front matter if set. Will override any Slug etc. - URL string - - // Used to create paginator links. - Addends string - - // The expanded permalink if defined for the section, ready to use. - ExpandedPermalink string - - // Some types cannot have uglyURLs, even if globally enabled, RSS being one example. - UglyURLs bool -} - -// createTargetPathDescriptor adapts a Page and the given output.Format into -// a targetPathDescriptor. This descriptor can then be used to create paths -// and URLs for this Page. -func (p *Page) createTargetPathDescriptor(t output.Format) (targetPathDescriptor, error) { - if p.targetPathDescriptorPrototype == nil { - panic(fmt.Sprintf("Must run initTargetPathDescriptor() for page %q, kind %q", p.title, p.Kind)) - } - d := *p.targetPathDescriptorPrototype - d.Type = t - return d, nil -} - -func (p *Page) initTargetPathDescriptor() error { - d := &targetPathDescriptor{ - PathSpec: p.s.PathSpec, - Kind: p.Kind, - Sections: p.sections, - UglyURLs: p.s.Info.uglyURLs(p), - Dir: filepath.ToSlash(p.Dir()), - URL: p.frontMatterURL, - IsMultihost: p.s.owner.IsMultihost(), - } - - if p.Slug != "" { - d.BaseName = p.Slug - } else { - d.BaseName = p.TranslationBaseName() - } - - if p.shouldAddLanguagePrefix() { - d.LangPrefix = p.Lang() - } - - // Expand only KindPage and KindTaxonomy; don't expand other Kinds of Pages - // like KindSection or KindTaxonomyTerm because they are "shallower" and - // the permalink configuration values are likely to be redundant, e.g. - // naively expanding /category/:slug/ would give /category/categories/ for - // the "categories" KindTaxonomyTerm. - if p.Kind == KindPage || p.Kind == KindTaxonomy { - if override, ok := p.Site.Permalinks[p.Section()]; ok { - opath, err := override.Expand(p) - if err != nil { - return err - } - - opath, _ = url.QueryUnescape(opath) - opath = filepath.FromSlash(opath) - d.ExpandedPermalink = opath - } - } - - p.targetPathDescriptorPrototype = d - return nil - -} - -func (p *Page) initURLs() error { - if len(p.outputFormats) == 0 { - p.outputFormats = p.s.outputFormats[p.Kind] - } - target := filepath.ToSlash(p.createRelativeTargetPath()) - rel := p.s.PathSpec.URLizeFilename(target) - - var err error - f := p.outputFormats[0] - p.permalink, err = p.s.permalinkForOutputFormat(rel, f) - if err != nil { - return err - } - - p.relTargetPathBase = strings.TrimPrefix(strings.TrimSuffix(target, f.MediaType.FullSuffix()), "/") - if prefix := p.s.GetLanguagePrefix(); prefix != "" { - // Any language code in the path will be added later. - p.relTargetPathBase = strings.TrimPrefix(p.relTargetPathBase, prefix+"/") - } - p.relPermalink = p.s.PathSpec.PrependBasePath(rel, false) - p.layoutDescriptor = p.createLayoutDescriptor() - return nil -} - -func (p *Page) initPaths() error { - if err := p.initTargetPathDescriptor(); err != nil { - return err - } - if err := p.initURLs(); err != nil { - return err - } - return nil -} - -// createTargetPath creates the target filename for this Page for the given -// output.Format. Some additional URL parts can also be provided, the typical -// use case being pagination. -func (p *Page) createTargetPath(t output.Format, noLangPrefix bool, addends ...string) (string, error) { - d, err := p.createTargetPathDescriptor(t) - if err != nil { - return "", nil - } - - if noLangPrefix { - d.LangPrefix = "" - } - - if len(addends) > 0 { - d.Addends = filepath.Join(addends...) - } - - return createTargetPath(d), nil -} - -func createTargetPath(d targetPathDescriptor) string { - - pagePath := helpers.FilePathSeparator - - // The top level index files, i.e. the home page etc., needs - // the index base even when uglyURLs is enabled. - needsBase := true - - isUgly := d.UglyURLs && !d.Type.NoUgly - - if d.ExpandedPermalink == "" && d.BaseName != "" && d.BaseName == d.Type.BaseName { - isUgly = true - } - - if d.Kind != KindPage && d.URL == "" && len(d.Sections) > 0 { - if d.ExpandedPermalink != "" { - pagePath = filepath.Join(pagePath, d.ExpandedPermalink) - } else { - pagePath = filepath.Join(d.Sections...) - } - needsBase = false - } - - if d.Type.Path != "" { - pagePath = filepath.Join(pagePath, d.Type.Path) - } - - if d.Kind != KindHome && d.URL != "" { - if d.IsMultihost && d.LangPrefix != "" && !strings.HasPrefix(d.URL, "/"+d.LangPrefix) { - pagePath = filepath.Join(d.LangPrefix, pagePath, d.URL) - } else { - pagePath = filepath.Join(pagePath, d.URL) - } - - if d.Addends != "" { - pagePath = filepath.Join(pagePath, d.Addends) - } - - if strings.HasSuffix(d.URL, "/") || !strings.Contains(d.URL, ".") { - pagePath = filepath.Join(pagePath, d.Type.BaseName+d.Type.MediaType.FullSuffix()) - } - - } else if d.Kind == KindPage { - if d.ExpandedPermalink != "" { - pagePath = filepath.Join(pagePath, d.ExpandedPermalink) - - } else { - if d.Dir != "" { - pagePath = filepath.Join(pagePath, d.Dir) - } - if d.BaseName != "" { - pagePath = filepath.Join(pagePath, d.BaseName) - } - } - - if d.Addends != "" { - pagePath = filepath.Join(pagePath, d.Addends) - } - - if isUgly { - pagePath += d.Type.MediaType.FullSuffix() - } else { - pagePath = filepath.Join(pagePath, d.Type.BaseName+d.Type.MediaType.FullSuffix()) - } - - if d.LangPrefix != "" { - pagePath = filepath.Join(d.LangPrefix, pagePath) - } - } else { - if d.Addends != "" { - pagePath = filepath.Join(pagePath, d.Addends) - } - - needsBase = needsBase && d.Addends == "" - - // No permalink expansion etc. for node type pages (for now) - base := "" - - if needsBase || !isUgly { - base = helpers.FilePathSeparator + d.Type.BaseName - } - - pagePath += base + d.Type.MediaType.FullSuffix() - - if d.LangPrefix != "" { - pagePath = filepath.Join(d.LangPrefix, pagePath) - } - } - - pagePath = filepath.Join(helpers.FilePathSeparator, pagePath) - - // Note: MakePathSanitized will lower case the path if - // disablePathToLower isn't set. - return d.PathSpec.MakePathSanitized(pagePath) -} - -func (p *Page) createRelativeTargetPath() string { - - if len(p.outputFormats) == 0 { - if p.Kind == kindUnknown { - panic(fmt.Sprintf("Page %q has unknown kind", p.title)) - } - panic(fmt.Sprintf("Page %q missing output format(s)", p.title)) - } - - // Choose the main output format. In most cases, this will be HTML. - f := p.outputFormats[0] - - return p.createRelativeTargetPathForOutputFormat(f) - -} - -func (p *Page) createRelativePermalinkForOutputFormat(f output.Format) string { - return p.s.PathSpec.URLizeFilename(p.createRelativeTargetPathForOutputFormat(f)) -} - -func (p *Page) createRelativeTargetPathForOutputFormat(f output.Format) string { - tp, err := p.createTargetPath(f, p.s.owner.IsMultihost()) - - if err != nil { - p.s.Log.ERROR.Printf("Failed to create permalink for page %q: %s", p.FullFilePath(), err) - return "" - } - - // For /index.json etc. we must use the full path. - if f.MediaType.FullSuffix() == ".html" && filepath.Base(tp) == "index.html" { - tp = strings.TrimSuffix(tp, f.BaseFilename()) - } - - return tp -} diff --git a/hugolib/page_paths_test.go b/hugolib/page_paths_test.go deleted file mode 100644 index 8f8df6ec1..000000000 --- a/hugolib/page_paths_test.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "path/filepath" - "strings" - "testing" - - "github.com/gohugoio/hugo/media" - - "fmt" - - "github.com/gohugoio/hugo/output" -) - -func TestPageTargetPath(t *testing.T) { - - pathSpec := newTestDefaultPathSpec(t) - - noExtNoDelimMediaType := media.TextType - noExtNoDelimMediaType.Suffixes = []string{} - noExtNoDelimMediaType.Delimiter = "" - - // Netlify style _redirects - noExtDelimFormat := output.Format{ - Name: "NER", - MediaType: noExtNoDelimMediaType, - BaseName: "_redirects", - } - - for _, multiHost := range []bool{false, true} { - for _, langPrefix := range []string{"", "no"} { - for _, uglyURLs := range []bool{false, true} { - t.Run(fmt.Sprintf("multihost=%t,langPrefix=%q,uglyURLs=%t", multiHost, langPrefix, uglyURLs), - func(t *testing.T) { - - tests := []struct { - name string - d targetPathDescriptor - expected string - }{ - {"JSON home", targetPathDescriptor{Kind: KindHome, Type: output.JSONFormat}, "/index.json"}, - {"AMP home", targetPathDescriptor{Kind: KindHome, Type: output.AMPFormat}, "/amp/index.html"}, - {"HTML home", targetPathDescriptor{Kind: KindHome, BaseName: "_index", Type: output.HTMLFormat}, "/index.html"}, - {"Netlify redirects", targetPathDescriptor{Kind: KindHome, BaseName: "_index", Type: noExtDelimFormat}, "/_redirects"}, - {"HTML section list", targetPathDescriptor{ - Kind: KindSection, - Sections: []string{"sect1"}, - BaseName: "_index", - Type: output.HTMLFormat}, "/sect1/index.html"}, - {"HTML taxonomy list", targetPathDescriptor{ - Kind: KindTaxonomy, - Sections: []string{"tags", "hugo"}, - BaseName: "_index", - Type: output.HTMLFormat}, "/tags/hugo/index.html"}, - {"HTML taxonomy term", targetPathDescriptor{ - Kind: KindTaxonomy, - Sections: []string{"tags"}, - BaseName: "_index", - Type: output.HTMLFormat}, "/tags/index.html"}, - { - "HTML page", targetPathDescriptor{ - Kind: KindPage, - Dir: "/a/b", - BaseName: "mypage", - Sections: []string{"a"}, - Type: output.HTMLFormat}, "/a/b/mypage/index.html"}, - - { - "HTML page with index as base", targetPathDescriptor{ - Kind: KindPage, - Dir: "/a/b", - BaseName: "index", - Sections: []string{"a"}, - Type: output.HTMLFormat}, "/a/b/index.html"}, - - { - "HTML page with special chars", targetPathDescriptor{ - Kind: KindPage, - Dir: "/a/b", - BaseName: "My Page!", - Type: output.HTMLFormat}, "/a/b/My-Page/index.html"}, - {"RSS home", targetPathDescriptor{Kind: kindRSS, Type: output.RSSFormat}, "/index.xml"}, - {"RSS section list", targetPathDescriptor{ - Kind: kindRSS, - Sections: []string{"sect1"}, - Type: output.RSSFormat}, "/sect1/index.xml"}, - { - "AMP page", targetPathDescriptor{ - Kind: KindPage, - Dir: "/a/b/c", - BaseName: "myamp", - Type: output.AMPFormat}, "/amp/a/b/c/myamp/index.html"}, - { - "AMP page with URL with suffix", targetPathDescriptor{ - Kind: KindPage, - Dir: "/sect/", - BaseName: "mypage", - URL: "/some/other/url.xhtml", - Type: output.HTMLFormat}, "/some/other/url.xhtml"}, - { - "JSON page with URL without suffix", targetPathDescriptor{ - Kind: KindPage, - Dir: "/sect/", - BaseName: "mypage", - URL: "/some/other/path/", - Type: output.JSONFormat}, "/some/other/path/index.json"}, - { - "JSON page with URL without suffix and no trailing slash", targetPathDescriptor{ - Kind: KindPage, - Dir: "/sect/", - BaseName: "mypage", - URL: "/some/other/path", - Type: output.JSONFormat}, "/some/other/path/index.json"}, - { - "HTML page with expanded permalink", targetPathDescriptor{ - Kind: KindPage, - Dir: "/a/b", - BaseName: "mypage", - ExpandedPermalink: "/2017/10/my-title", - Type: output.HTMLFormat}, "/2017/10/my-title/index.html"}, - { - "Paginated HTML home", targetPathDescriptor{ - Kind: KindHome, - BaseName: "_index", - Type: output.HTMLFormat, - Addends: "page/3"}, "/page/3/index.html"}, - { - "Paginated Taxonomy list", targetPathDescriptor{ - Kind: KindTaxonomy, - BaseName: "_index", - Sections: []string{"tags", "hugo"}, - Type: output.HTMLFormat, - Addends: "page/3"}, "/tags/hugo/page/3/index.html"}, - { - "Regular page with addend", targetPathDescriptor{ - Kind: KindPage, - Dir: "/a/b", - BaseName: "mypage", - Addends: "c/d/e", - Type: output.HTMLFormat}, "/a/b/mypage/c/d/e/index.html"}, - } - - for i, test := range tests { - test.d.PathSpec = pathSpec - test.d.UglyURLs = uglyURLs - test.d.LangPrefix = langPrefix - test.d.IsMultihost = multiHost - test.d.Dir = filepath.FromSlash(test.d.Dir) - isUgly := uglyURLs && !test.d.Type.NoUgly - - expected := test.expected - - // TODO(bep) simplify - if test.d.Kind == KindPage && test.d.BaseName == test.d.Type.BaseName { - - } else if test.d.Kind == KindHome && test.d.Type.Path != "" { - } else if (!strings.HasPrefix(expected, "/index") || test.d.Addends != "") && test.d.URL == "" && isUgly { - expected = strings.Replace(expected, - "/"+test.d.Type.BaseName+"."+test.d.Type.MediaType.Suffix(), - "."+test.d.Type.MediaType.Suffix(), -1) - } - - if test.d.LangPrefix != "" && !(test.d.Kind == KindPage && test.d.URL != "") { - expected = "/" + test.d.LangPrefix + expected - } else if multiHost && test.d.LangPrefix != "" && test.d.URL != "" { - expected = "/" + test.d.LangPrefix + expected - } - - expected = filepath.FromSlash(expected) - - pagePath := createTargetPath(test.d) - - if pagePath != expected { - t.Fatalf("[%d] [%s] targetPath expected %q, got: %q", i, test.name, expected, pagePath) - } - } - }) - } - } - } -} diff --git a/hugolib/page_permalink_test.go b/hugolib/page_permalink_test.go index 76b0b8635..ed6eb11e3 100644 --- a/hugolib/page_permalink_test.go +++ b/hugolib/page_permalink_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -63,40 +63,44 @@ func TestPermalink(t *testing.T) { } for i, test := range tests { + t.Run(fmt.Sprintf("%s-%d", test.file, i), func(t *testing.T) { - cfg, fs := newTestCfg() + cfg, fs := newTestCfg() - cfg.Set("uglyURLs", test.uglyURLs) - cfg.Set("canonifyURLs", test.canonifyURLs) - cfg.Set("baseURL", test.base) + cfg.Set("uglyURLs", test.uglyURLs) + cfg.Set("canonifyURLs", test.canonifyURLs) + cfg.Set("baseURL", test.base) - pageContent := fmt.Sprintf(`--- + pageContent := fmt.Sprintf(`--- title: Page slug: %q url: %q +output: ["HTML"] --- Content `, test.slug, test.url) - writeSource(t, fs, filepath.Join("content", filepath.FromSlash(test.file)), pageContent) + writeSource(t, fs, filepath.Join("content", filepath.FromSlash(test.file)), pageContent) - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - require.Len(t, s.RegularPages, 1) + s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) + require.Len(t, s.RegularPages(), 1) - p := s.RegularPages[0] + p := s.RegularPages()[0] - u := p.Permalink() + u := p.Permalink() - expected := test.expectedAbs - if u != expected { - t.Fatalf("[%d] Expected abs url: %s, got: %s", i, expected, u) - } + expected := test.expectedAbs + if u != expected { + t.Fatalf("[%d] Expected abs url: %s, got: %s", i, expected, u) + } - u = p.RelPermalink() + u = p.RelPermalink() - expected = test.expectedRel - if u != expected { - t.Errorf("[%d] Expected rel url: %s, got: %s", i, expected, u) - } + expected = test.expectedRel + if u != expected { + t.Errorf("[%d] Expected rel url: %s, got: %s", i, expected, u) + } + }) } + } diff --git a/hugolib/page_taxonomy_test.go b/hugolib/page_taxonomy_test.go deleted file mode 100644 index ed1d2565d..000000000 --- a/hugolib/page_taxonomy_test.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "reflect" - "strings" - "testing" -) - -var pageYamlWithTaxonomiesA = `--- -tags: ['a', 'B', 'c'] -categories: 'd' ---- -YAML frontmatter with tags and categories taxonomy.` - -var pageYamlWithTaxonomiesB = `--- -tags: - - "a" - - "B" - - "c" -categories: 'd' ---- -YAML frontmatter with tags and categories taxonomy.` - -var pageYamlWithTaxonomiesC = `--- -tags: 'E' -categories: 'd' ---- -YAML frontmatter with tags and categories taxonomy.` - -var pageJSONWithTaxonomies = `{ - "categories": "D", - "tags": [ - "a", - "b", - "c" - ] -} -JSON Front Matter with tags and categories` - -var pageTomlWithTaxonomies = `+++ -tags = [ "a", "B", "c" ] -categories = "d" -+++ -TOML Front Matter with tags and categories` - -func TestParseTaxonomies(t *testing.T) { - t.Parallel() - for _, test := range []string{pageTomlWithTaxonomies, - pageJSONWithTaxonomies, - pageYamlWithTaxonomiesA, - pageYamlWithTaxonomiesB, - pageYamlWithTaxonomiesC, - } { - - s := newTestSite(t) - p, _ := s.NewPage("page/with/taxonomy") - _, err := p.ReadFrom(strings.NewReader(test)) - if err != nil { - t.Fatalf("Failed parsing %q: %s", test, err) - } - - param := p.getParamToLower("tags") - - if params, ok := param.([]string); ok { - expected := []string{"a", "b", "c"} - if !reflect.DeepEqual(params, expected) { - t.Errorf("Expected %s: got: %s", expected, params) - } - } else if params, ok := param.(string); ok { - expected := "e" - if params != expected { - t.Errorf("Expected %s: got: %s", expected, params) - } - } - - param = p.getParamToLower("categories") - singleparam := param.(string) - - if singleparam != "d" { - t.Fatalf("Expected: d, got: %s", singleparam) - } - } -} diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 1db1d3522..5e9ac696c 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,88 +14,34 @@ package hugolib import ( - "bytes" "fmt" "html/template" "os" "path/filepath" - "reflect" - "sort" "strings" "testing" "time" "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/afero" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource" + + "github.com/spf13/afero" "github.com/spf13/viper" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" - "github.com/spf13/cast" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -var emptyPage = "" - const ( - homePage = "---\ntitle: Home\n---\nHome Page Content\n" - simplePage = "---\ntitle: Simple\n---\nSimple Page\n" - renderNoFrontmatter = "<!doctype><html><head></head><body>This is a test</body></html>" - contentNoFrontmatter = "Page without front matter.\n" - contentWithCommentedFrontmatter = "<!--\n+++\ntitle = \"Network configuration\"\ndescription = \"Docker networking\"\nkeywords = [\"network\"]\n[menu.main]\nparent= \"smn_administrate\"\n+++\n-->\n\n# Network configuration\n\n##\nSummary" - contentWithCommentedTextFrontmatter = "<!--[metaData]>\n+++\ntitle = \"Network configuration\"\ndescription = \"Docker networking\"\nkeywords = [\"network\"]\n[menu.main]\nparent= \"smn_administrate\"\n+++\n<![end-metadata]-->\n\n# Network configuration\n\n##\nSummary" - contentWithCommentedLongFrontmatter = "<!--[metaData123456789012345678901234567890]>\n+++\ntitle = \"Network configuration\"\ndescription = \"Docker networking\"\nkeywords = [\"network\"]\n[menu.main]\nparent= \"smn_administrate\"\n+++\n<![end-metadata]-->\n\n# Network configuration\n\n##\nSummary" - contentWithCommentedLong2Frontmatter = "<!--[metaData]>\n+++\ntitle = \"Network configuration\"\ndescription = \"Docker networking\"\nkeywords = [\"network\"]\n[menu.main]\nparent= \"smn_administrate\"\n+++\n<![end-metadata123456789012345678901234567890]-->\n\n# Network configuration\n\n##\nSummary" - invalidFrontmatterShortDelim = ` --- -title: Short delim start ---- -Short Delim -` - - invalidFrontmatterShortDelimEnding = ` ---- -title: Short delim ending --- -Short Delim -` + homePage = "---\ntitle: Home\n---\nHome Page Content\n" + simplePage = "---\ntitle: Simple\n---\nSimple Page\n" - invalidFrontmatterLadingWs = ` - - --- -title: Leading WS ---- -Leading -` - - simplePageJSON = ` -{ -"title": "spf13-vim 3.0 release and new website", -"description": "spf13-vim is a cross platform distribution of vim plugins and resources for Vim.", -"tags": [ ".vimrc", "plugins", "spf13-vim", "VIm" ], -"date": "2012-04-06", -"categories": [ - "Development", - "VIM" -], -"slug": "-spf13-vim-3-0-release-and-new-website-" -} - -Content of the file goes Here -` - - simplePageRFC3339Date = "---\ntitle: RFC3339 Date\ndate: \"2013-05-17T16:59:30Z\"\n---\nrfc3339 content" - simplePageJSONMultiple = ` -{ - "title": "foobar", - "customData": { "foo": "bar" }, - "date": "2012-08-06" -} -Some text -` + simplePageRFC3339Date = "---\ntitle: RFC3339 Date\ndate: \"2013-05-17T16:59:30Z\"\n---\nrfc3339 content" simplePageWithSummaryDelimiter = `--- title: Simple @@ -137,14 +83,6 @@ Summary Same Line<!--more--> Some more text ` - simplePageWithSummaryDelimiterOnlySummary = `--- -title: Simple ---- -Summary text - -<!--more--> -` - simplePageWithAllCJKRunes = `--- title: Simple --- @@ -334,156 +272,17 @@ date: '2013-10-15T06:16:13' UTF8 Page With Date` ) -var pageWithVariousFrontmatterTypes = `+++ -a_string = "bar" -an_integer = 1 -a_float = 1.3 -a_bool = false -a_date = 1979-05-27T07:32:00Z - -[a_table] -a_key = "a_value" -+++ -Front Matter with various frontmatter types` - -var pageWithCalendarYAMLFrontmatter = `--- -type: calendar -weeks: - - - start: "Jan 5" - days: - - activity: class - room: EN1000 - - activity: lab - - activity: class - - activity: lab - - activity: class - - - start: "Jan 12" - days: - - activity: class - - activity: lab - - activity: class - - activity: lab - - activity: exam ---- - -Hi. -` - -var pageWithCalendarJSONFrontmatter = `{ - "type": "calendar", - "weeks": [ - { - "start": "Jan 5", - "days": [ - { "activity": "class", "room": "EN1000" }, - { "activity": "lab" }, - { "activity": "class" }, - { "activity": "lab" }, - { "activity": "class" } - ] - }, - { - "start": "Jan 12", - "days": [ - { "activity": "class" }, - { "activity": "lab" }, - { "activity": "class" }, - { "activity": "lab" }, - { "activity": "exam" } - ] - } - ] -} - -Hi. -` - -var pageWithCalendarTOMLFrontmatter = `+++ -type = "calendar" - -[[weeks]] -start = "Jan 5" - -[[weeks.days]] -activity = "class" -room = "EN1000" - -[[weeks.days]] -activity = "lab" - -[[weeks.days]] -activity = "class" - -[[weeks.days]] -activity = "lab" - -[[weeks.days]] -activity = "class" - -[[weeks]] -start = "Jan 12" - -[[weeks.days]] -activity = "class" - -[[weeks.days]] -activity = "lab" - -[[weeks.days]] -activity = "class" - -[[weeks.days]] -activity = "lab" - -[[weeks.days]] -activity = "exam" -+++ - -Hi. -` - -func checkError(t *testing.T, err error, expected string) { - if err == nil { - t.Fatalf("err is nil. Expected: %s", expected) - } - if !strings.Contains(err.Error(), expected) { - t.Errorf("err.Error() returned: '%s'. Expected: '%s'", err.Error(), expected) - } -} - -func TestDegenerateEmptyPageZeroLengthName(t *testing.T) { - t.Parallel() - s := newTestSite(t) - _, err := s.NewPage("") - if err == nil { - t.Fatalf("A zero length page name must return an error") - } - - checkError(t, err, "Zero length page name") -} - -func TestDegenerateEmptyPage(t *testing.T) { - t.Parallel() - s := newTestSite(t) - _, err := s.newPageFrom(strings.NewReader(emptyPage), "test") - if err != nil { - t.Fatalf("Empty files should not trigger an error. Should be able to touch a file while watching without erroring out.") - } -} - -func checkPageTitle(t *testing.T, page *Page, title string) { - if page.title != title { - t.Fatalf("Page title is: %s. Expected %s", page.title, title) +func checkPageTitle(t *testing.T, page page.Page, title string) { + if page.Title() != title { + t.Fatalf("Page title is: %s. Expected %s", page.Title(), title) } } -func checkPageContent(t *testing.T, page *Page, content string, msg ...interface{}) { - a := normalizeContent(content) - b := normalizeContent(string(page.content())) +func checkPageContent(t *testing.T, page page.Page, expected string, msg ...interface{}) { + a := normalizeContent(expected) + b := normalizeContent(content(page)) if a != b { - t.Log(trace()) + t.Log(stackTrace()) t.Fatalf("Page content is:\n%q\nExpected:\n%q (%q)", b, a, msg) } } @@ -499,42 +298,29 @@ func normalizeContent(c string) string { return strings.TrimSpace(norm) } -func checkPageTOC(t *testing.T, page *Page, toc string) { - if page.TableOfContents != template.HTML(toc) { - t.Fatalf("Page TableOfContents is: %q.\nExpected %q", page.TableOfContents, toc) +func checkPageTOC(t *testing.T, page page.Page, toc string) { + if page.TableOfContents() != template.HTML(toc) { + t.Fatalf("Page TableOfContents is: %q.\nExpected %q", page.TableOfContents(), toc) } } -func checkPageSummary(t *testing.T, page *Page, summary string, msg ...interface{}) { - a := normalizeContent(string(page.summary)) +func checkPageSummary(t *testing.T, page page.Page, summary string, msg ...interface{}) { + a := normalizeContent(string(page.Summary())) b := normalizeContent(summary) if a != b { t.Fatalf("Page summary is:\n%q.\nExpected\n%q (%q)", a, b, msg) } } -func checkPageType(t *testing.T, page *Page, pageType string) { +func checkPageType(t *testing.T, page page.Page, pageType string) { if page.Type() != pageType { t.Fatalf("Page type is: %s. Expected: %s", page.Type(), pageType) } } -func checkPageDate(t *testing.T, page *Page, time time.Time) { - if page.Date != time { - t.Fatalf("Page date is: %s. Expected: %s", page.Date, time) - } -} - -func checkTruncation(t *testing.T, page *Page, shouldBe bool, msg string) { - if page.Summary() == "" { - t.Fatal("page has no summary, can not check truncation") - } - if page.truncated != shouldBe { - if shouldBe { - t.Fatalf("page wasn't truncated: %s", msg) - } else { - t.Fatalf("page was truncated: %s", msg) - } +func checkPageDate(t *testing.T, page page.Page, time time.Time) { + if page.Date() != time { + t.Fatalf("Page date is: %s. Expected: %s", page.Date(), time) } } @@ -562,7 +348,7 @@ func normalizeExpected(ext, str string) string { } func testAllMarkdownEnginesForPages(t *testing.T, - assertFunc func(t *testing.T, ext string, pages Pages), settings map[string]interface{}, pageSources ...string) { + assertFunc func(t *testing.T, ext string, pages page.Pages), settings map[string]interface{}, pageSources ...string) { engines := []struct { ext string @@ -607,33 +393,93 @@ func testAllMarkdownEnginesForPages(t *testing.T, s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - require.Len(t, s.RegularPages, len(pageSources)) + require.Len(t, s.RegularPages(), len(pageSources)) - assertFunc(t, e.ext, s.RegularPages) + assertFunc(t, e.ext, s.RegularPages()) home, err := s.Info.Home() require.NoError(t, err) require.NotNil(t, home) - require.Equal(t, homePath, home.Path()) - require.Contains(t, home.content(), "Home Page Content") + require.Equal(t, homePath, home.File().Path()) + require.Contains(t, content(home), "Home Page Content") + + } +} + +// Issue #1076 +func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) { + t.Parallel() + cfg, fs := newTestCfg() + + writeSource(t, fs, filepath.Join("content", "simple.md"), simplePageWithSummaryDelimiterAndMarkdownThatCrossesBorder) + + s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) + + require.Len(t, s.RegularPages(), 1) + + p := s.RegularPages()[0] + + if p.Summary() != template.HTML( + "<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a href=\"#fn:1\">1</a></sup></p>") { + t.Fatalf("Got summary:\n%q", p.Summary()) + } + + c := content(p) + if c != "<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a href=\"#fn:1\">1</a></sup></p>\n\n<div class=\"footnotes\">\n\n<hr />\n\n<ol>\n<li id=\"fn:1\">Many people say so.\n <a class=\"footnote-return\" href=\"#fnref:1\"><sup>[return]</sup></a></li>\n</ol>\n</div>" { + t.Fatalf("Got content:\n%q", c) + } +} + +func TestPageDatesAllKinds(t *testing.T) { + t.Parallel() + assert := assert.New(t) + + pageContent := ` +--- +title: Page +date: 2017-01-15 +tags: ["hugo"] +categories: ["cool stuff"] +--- +` + + b := newTestSitesBuilder(t) + b.WithSimpleConfigFile().WithContent("page.md", pageContent) + b.WithSimpleConfigFile().WithContent("blog/page.md", pageContent) + + b.CreateSites().Build(BuildCfg{}) + + assert.Equal(1, len(b.H.Sites)) + s := b.H.Sites[0] + + checkDate := func(t time.Time, msg string) { + assert.Equal(2017, t.Year(), msg) + } + + checkDated := func(d resource.Dated, msg string) { + checkDate(d.Date(), "date: "+msg) + checkDate(d.Lastmod(), "lastmod: "+msg) } + for _, p := range s.Pages() { + checkDated(p, p.Kind()) + } + checkDate(s.Info.LastChange(), "site") } func TestCreateNewPage(t *testing.T) { t.Parallel() - assertFunc := func(t *testing.T, ext string, pages Pages) { + assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] // issue #2290: Path is relative to the content dir and will continue to be so. - require.Equal(t, filepath.FromSlash(fmt.Sprintf("p0.%s", ext)), p.Path()) + require.Equal(t, filepath.FromSlash(fmt.Sprintf("p0.%s", ext)), p.File().Path()) assert.False(t, p.IsHome()) checkPageTitle(t, p, "Simple") checkPageContent(t, p, normalizeExpected(ext, "<p>Simple Page</p>\n")) checkPageSummary(t, p, "Simple Page") checkPageType(t, p, "page") - checkTruncation(t, p, false, "simple short page") } settings := map[string]interface{}{ @@ -645,43 +491,17 @@ func TestCreateNewPage(t *testing.T) { func TestPageWithDelimiter(t *testing.T) { t.Parallel() - assertFunc := func(t *testing.T, ext string, pages Pages) { + assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] checkPageTitle(t, p, "Simple") checkPageContent(t, p, normalizeExpected(ext, "<p>Summary Next Line</p>\n\n<p>Some more text</p>\n"), ext) checkPageSummary(t, p, normalizeExpected(ext, "<p>Summary Next Line</p>"), ext) checkPageType(t, p, "page") - checkTruncation(t, p, true, "page with summary delimiter") } testAllMarkdownEnginesForPages(t, assertFunc, nil, simplePageWithSummaryDelimiter) } -// Issue #1076 -func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) { - t.Parallel() - cfg, fs := newTestCfg() - - writeSource(t, fs, filepath.Join("content", "simple.md"), simplePageWithSummaryDelimiterAndMarkdownThatCrossesBorder) - - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - - require.Len(t, s.RegularPages, 1) - - p := s.RegularPages[0] - - if p.Summary() != template.HTML( - "<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a href=\"#fn:1\">1</a></sup></p>") { - t.Fatalf("Got summary:\n%q", p.Summary()) - } - - if p.content() != template.HTML( - "<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a href=\"#fn:1\">1</a></sup></p>\n\n<div class=\"footnotes\">\n\n<hr />\n\n<ol>\n<li id=\"fn:1\">Many people say so.\n <a class=\"footnote-return\" href=\"#fnref:1\"><sup>[return]</sup></a></li>\n</ol>\n</div>") { - - t.Fatalf("Got content:\n%q", p.content()) - } -} - // Issue #3854 // Also see https://github.com/gohugoio/hugo/issues/3977 func TestPageWithDateFields(t *testing.T) { @@ -693,8 +513,8 @@ weight: %d --- Simple Page With Some Date` - hasDate := func(p *Page) bool { - return p.Date.Year() == 2017 + hasDate := func(p page.Page) bool { + return p.Date().Year() == 2017 } datePage := func(field string, weight int) string { @@ -702,7 +522,7 @@ Simple Page With Some Date` } t.Parallel() - assertFunc := func(t *testing.T, ext string, pages Pages) { + assertFunc := func(t *testing.T, ext string, pages page.Pages) { assert.True(len(pages) > 0) for _, p := range pages { assert.True(hasDate(p)) @@ -733,8 +553,8 @@ title: Raw s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - require.Len(t, s.RegularPages, 1) - p := s.RegularPages[0] + require.Len(t, s.RegularPages(), 1) + p := s.RegularPages()[0] require.Equal(t, p.RawContent(), "**Raw**") @@ -742,7 +562,7 @@ title: Raw func TestPageWithShortCodeInSummary(t *testing.T) { t.Parallel() - assertFunc := func(t *testing.T, ext string, pages Pages) { + assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] checkPageTitle(t, p, "Simple") checkPageContent(t, p, normalizeExpected(ext, "<p>Summary Next Line. <figure> <img src=\"/not/real\"/> </figure> . More text here.</p><p>Some more text</p>")) @@ -755,7 +575,7 @@ func TestPageWithShortCodeInSummary(t *testing.T) { func TestPageWithEmbeddedScriptTag(t *testing.T) { t.Parallel() - assertFunc := func(t *testing.T, ext string, pages Pages) { + assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] if ext == "ad" || ext == "rst" { // TOD(bep) @@ -775,9 +595,9 @@ func TestPageWithAdditionalExtension(t *testing.T) { s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - require.Len(t, s.RegularPages, 1) + require.Len(t, s.RegularPages(), 1) - p := s.RegularPages[0] + p := s.RegularPages()[0] checkPageContent(t, p, "<p>first line.<br />\nsecond line.</p>\n\n<p>fourth line.</p>\n") } @@ -790,9 +610,9 @@ func TestTableOfContents(t *testing.T) { s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - require.Len(t, s.RegularPages, 1) + require.Len(t, s.RegularPages(), 1) - p := s.RegularPages[0] + p := s.RegularPages()[0] checkPageContent(t, p, "\n\n<p>For some moments the old man did not reply. He stood with bowed head, buried in deep thought. But at last he spoke.</p>\n\n<h2 id=\"aa\">AA</h2>\n\n<p>I have no idea, of course, how long it took me to reach the limit of the plain,\nbut at last I entered the foothills, following a pretty little canyon upward\ntoward the mountains. Beside me frolicked a laughing brooklet, hurrying upon\nits noisy way down to the silent sea. In its quieter pools I discovered many\nsmall fish, of four-or five-pound weight I should imagine. In appearance,\nexcept as to size and color, they were not unlike the whale of our own seas. As\nI watched them playing about I discovered, not only that they suckled their\nyoung, but that at intervals they rose to the surface to breathe as well as to\nfeed upon certain grasses and a strange, scarlet lichen which grew upon the\nrocks just above the water line.</p>\n\n<h3 id=\"aaa\">AAA</h3>\n\n<p>I remember I felt an extraordinary persuasion that I was being played with,\nthat presently, when I was upon the very verge of safety, this mysterious\ndeath–as swift as the passage of light–would leap after me from the pit about\nthe cylinder and strike me down. ## BB</p>\n\n<h3 id=\"bbb\">BBB</h3>\n\n<p>“You’re a great Granser,” he cried delightedly, “always making believe them little marks mean something.”</p>\n") checkPageTOC(t, p, "<nav id=\"TableOfContents\">\n<ul>\n<li>\n<ul>\n<li><a href=\"#aa\">AA</a>\n<ul>\n<li><a href=\"#aaa\">AAA</a></li>\n<li><a href=\"#bbb\">BBB</a></li>\n</ul></li>\n</ul></li>\n</ul>\n</nav>") @@ -800,7 +620,7 @@ func TestTableOfContents(t *testing.T) { func TestPageWithMoreTag(t *testing.T) { t.Parallel() - assertFunc := func(t *testing.T, ext string, pages Pages) { + assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] checkPageTitle(t, p, "Simple") checkPageContent(t, p, normalizeExpected(ext, "<p>Summary Same Line</p>\n\n<p>Some more text</p>\n")) @@ -812,20 +632,10 @@ func TestPageWithMoreTag(t *testing.T) { testAllMarkdownEnginesForPages(t, assertFunc, nil, simplePageWithSummaryDelimiterSameLine) } -func TestPageWithMoreTagOnlySummary(t *testing.T) { - - assertFunc := func(t *testing.T, ext string, pages Pages) { - p := pages[0] - checkTruncation(t, p, false, "page with summary delimiter at end") - } - - testAllMarkdownEnginesForPages(t, assertFunc, nil, simplePageWithSummaryDelimiterOnlySummary) -} - // #2973 func TestSummaryWithHTMLTagsOnNextLine(t *testing.T) { - assertFunc := func(t *testing.T, ext string, pages Pages) { + assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] require.Contains(t, p.Summary(), "Happy new year everyone!") require.NotContains(t, p.Summary(), "User interface") @@ -853,9 +663,9 @@ func TestPageWithDate(t *testing.T) { s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - require.Len(t, s.RegularPages, 1) + require.Len(t, s.RegularPages(), 1) - p := s.RegularPages[0] + p := s.RegularPages()[0] d, _ := time.Parse(time.RFC3339, "2013-05-17T16:59:30Z") checkPageDate(t, p, d) @@ -905,16 +715,16 @@ func TestPageWithLastmodFromGitInfo(t *testing.T) { require.NoError(t, h.Build(BuildCfg{SkipRender: true})) enSite := h.Sites[0] - assrt.Len(enSite.RegularPages, 1) + assrt.Len(enSite.RegularPages(), 1) // 2018-03-11 is the Git author date for testsite/content/first-post.md - assrt.Equal("2018-03-11", enSite.RegularPages[0].Lastmod.Format("2006-01-02")) + assrt.Equal("2018-03-11", enSite.RegularPages()[0].Lastmod().Format("2006-01-02")) nnSite := h.Sites[1] - assrt.Len(nnSite.RegularPages, 1) + assrt.Len(nnSite.RegularPages(), 1) // 2018-08-11 is the Git author date for testsite/content_nn/first-post.md - assrt.Equal("2018-08-11", nnSite.RegularPages[0].Lastmod.Format("2006-01-02")) + assrt.Equal("2018-08-11", nnSite.RegularPages()[0].Lastmod().Format("2006-01-02")) } @@ -953,28 +763,28 @@ Content s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - assrt.Len(s.RegularPages, 2) + assrt.Len(s.RegularPages(), 2) - noSlug := s.RegularPages[0] - slug := s.RegularPages[1] + noSlug := s.RegularPages()[0] + slug := s.RegularPages()[1] - assrt.Equal(28, noSlug.Lastmod.Day()) + assrt.Equal(28, noSlug.Lastmod().Day()) switch strings.ToLower(dateHandler) { case ":filename": - assrt.False(noSlug.Date.IsZero()) - assrt.False(slug.Date.IsZero()) - assrt.Equal(2012, noSlug.Date.Year()) - assrt.Equal(2012, slug.Date.Year()) - assrt.Equal("noslug", noSlug.Slug) - assrt.Equal("aslug", slug.Slug) + assrt.False(noSlug.Date().IsZero()) + assrt.False(slug.Date().IsZero()) + assrt.Equal(2012, noSlug.Date().Year()) + assrt.Equal(2012, slug.Date().Year()) + assrt.Equal("noslug", noSlug.Slug()) + assrt.Equal("aslug", slug.Slug()) case ":filemodtime": - assrt.Equal(c1fi.ModTime().Year(), noSlug.Date.Year()) - assrt.Equal(c2fi.ModTime().Year(), slug.Date.Year()) + assrt.Equal(c1fi.ModTime().Year(), noSlug.Date().Year()) + assrt.Equal(c2fi.ModTime().Year(), slug.Date().Year()) fallthrough default: - assrt.Equal("", noSlug.Slug) - assrt.Equal("aslug", slug.Slug) + assrt.Equal("", noSlug.Slug()) + assrt.Equal("aslug", slug.Slug()) } }) @@ -984,10 +794,10 @@ Content func TestWordCountWithAllCJKRunesWithoutHasCJKLanguage(t *testing.T) { t.Parallel() - assertFunc := func(t *testing.T, ext string, pages Pages) { + assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] if p.WordCount() != 8 { - t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 8, p.WordCount()) + t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 8, p.WordCount()) } } @@ -998,10 +808,10 @@ func TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) { t.Parallel() settings := map[string]interface{}{"hasCJKLanguage": true} - assertFunc := func(t *testing.T, ext string, pages Pages) { + assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] if p.WordCount() != 15 { - t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 15, p.WordCount()) + t.Fatalf("[%s] incorrect word count, expected %v, got %v", ext, 15, p.WordCount()) } } testAllMarkdownEnginesForPages(t, assertFunc, settings, simplePageWithAllCJKRunes) @@ -1011,15 +821,15 @@ func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) { t.Parallel() settings := map[string]interface{}{"hasCJKLanguage": true} - assertFunc := func(t *testing.T, ext string, pages Pages) { + assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] if p.WordCount() != 74 { - t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 74, p.WordCount()) + t.Fatalf("[%s] incorrect word count, expected %v, got %v", ext, 74, p.WordCount()) } - if p.summary != simplePageWithMainEnglishWithCJKRunesSummary { - t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.plain, - simplePageWithMainEnglishWithCJKRunesSummary, p.summary) + if p.Summary() != simplePageWithMainEnglishWithCJKRunesSummary { + t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.Plain(), + simplePageWithMainEnglishWithCJKRunesSummary, p.Summary()) } } @@ -1032,15 +842,15 @@ func TestWordCountWithIsCJKLanguageFalse(t *testing.T) { "hasCJKLanguage": true, } - assertFunc := func(t *testing.T, ext string, pages Pages) { + assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] if p.WordCount() != 75 { - t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 74, p.WordCount()) + t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.Plain(), 74, p.WordCount()) } - if p.summary != simplePageWithIsCJKLanguageFalseSummary { - t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.plain, - simplePageWithIsCJKLanguageFalseSummary, p.summary) + if p.Summary() != simplePageWithIsCJKLanguageFalseSummary { + t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.Plain(), + simplePageWithIsCJKLanguageFalseSummary, p.Summary()) } } @@ -1050,7 +860,7 @@ func TestWordCountWithIsCJKLanguageFalse(t *testing.T) { func TestWordCount(t *testing.T) { t.Parallel() - assertFunc := func(t *testing.T, ext string, pages Pages) { + assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] if p.WordCount() != 483 { t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 483, p.WordCount()) @@ -1064,163 +874,11 @@ func TestWordCount(t *testing.T) { t.Fatalf("[%s] incorrect min read. expected %v, got %v", ext, 3, p.ReadingTime()) } - checkTruncation(t, p, true, "long page") } testAllMarkdownEnginesForPages(t, assertFunc, nil, simplePageWithLongContent) } -func TestCreatePage(t *testing.T) { - t.Parallel() - var tests = []struct { - r string - }{ - {simplePageJSON}, - {simplePageJSONMultiple}, - //{strings.NewReader(SIMPLE_PAGE_JSON_COMPACT)}, - } - - for i, test := range tests { - s := newTestSite(t) - p, _ := s.NewPage("page") - if _, err := p.ReadFrom(strings.NewReader(test.r)); err != nil { - t.Fatalf("[%d] Unable to parse page: %s", i, err) - } - } -} - -func TestDegenerateInvalidFrontMatterShortDelim(t *testing.T) { - t.Parallel() - var tests = []struct { - r string - err string - }{ - {invalidFrontmatterShortDelimEnding, "EOF looking for end YAML front matter delimiter"}, - } - for _, test := range tests { - s := newTestSite(t) - p, _ := s.NewPage("invalid/front/matter/short/delim") - _, err := p.ReadFrom(strings.NewReader(test.r)) - checkError(t, err, test.err) - } -} - -func TestShouldRenderContent(t *testing.T) { - t.Parallel() - assert := require.New(t) - - var tests = []struct { - text string - render bool - }{ - {contentNoFrontmatter, true}, - {renderNoFrontmatter, false}, - {contentWithCommentedFrontmatter, true}, - {contentWithCommentedTextFrontmatter, true}, - {contentWithCommentedLongFrontmatter, true}, - {contentWithCommentedLong2Frontmatter, true}, - } - - for i, test := range tests { - s := newTestSite(t) - p, _ := s.NewPage("render/front/matter") - _, err := p.ReadFrom(strings.NewReader(test.text)) - msg := fmt.Sprintf("test %d", i) - assert.NoError(err, msg) - assert.Equal(test.render, p.IsRenderable(), msg) - } -} - -// Issue #768 -func TestCalendarParamsVariants(t *testing.T) { - t.Parallel() - s := newTestSite(t) - pageJSON, _ := s.NewPage("test/fileJSON.md") - _, _ = pageJSON.ReadFrom(strings.NewReader(pageWithCalendarJSONFrontmatter)) - - pageYAML, _ := s.NewPage("test/fileYAML.md") - _, _ = pageYAML.ReadFrom(strings.NewReader(pageWithCalendarYAMLFrontmatter)) - - pageTOML, _ := s.NewPage("test/fileTOML.md") - _, _ = pageTOML.ReadFrom(strings.NewReader(pageWithCalendarTOMLFrontmatter)) - - assert.True(t, compareObjects(pageJSON.params, pageYAML.params)) - assert.True(t, compareObjects(pageJSON.params, pageTOML.params)) - -} - -func TestDifferentFrontMatterVarTypes(t *testing.T) { - t.Parallel() - s := newTestSite(t) - page, _ := s.NewPage("test/file1.md") - _, _ = page.ReadFrom(strings.NewReader(pageWithVariousFrontmatterTypes)) - - dateval, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z") - if page.getParamToLower("a_string") != "bar" { - t.Errorf("frontmatter not handling strings correctly should be %s, got: %s", "bar", page.getParamToLower("a_string")) - } - if page.getParamToLower("an_integer") != 1 { - t.Errorf("frontmatter not handling ints correctly should be %s, got: %s", "1", page.getParamToLower("an_integer")) - } - if page.getParamToLower("a_float") != 1.3 { - t.Errorf("frontmatter not handling floats correctly should be %f, got: %s", 1.3, page.getParamToLower("a_float")) - } - if page.getParamToLower("a_bool") != false { - t.Errorf("frontmatter not handling bools correctly should be %t, got: %s", false, page.getParamToLower("a_bool")) - } - if page.getParamToLower("a_date") != dateval { - t.Errorf("frontmatter not handling dates correctly should be %s, got: %s", dateval, page.getParamToLower("a_date")) - } - param := page.getParamToLower("a_table") - if param == nil { - t.Errorf("frontmatter not handling tables correctly should be type of %v, got: type of %v", reflect.TypeOf(page.params["a_table"]), reflect.TypeOf(param)) - } - if cast.ToStringMap(param)["a_key"] != "a_value" { - t.Errorf("frontmatter not handling values inside a table correctly should be %s, got: %s", "a_value", cast.ToStringMap(page.params["a_table"])["a_key"]) - } -} - -func TestDegenerateInvalidFrontMatterLeadingWhitespace(t *testing.T) { - t.Parallel() - s := newTestSite(t) - p, _ := s.NewPage("invalid/front/matter/leading/ws") - _, err := p.ReadFrom(strings.NewReader(invalidFrontmatterLadingWs)) - if err != nil { - t.Fatalf("Unable to parse front matter given leading whitespace: %s", err) - } -} - -func TestSectionEvaluation(t *testing.T) { - t.Parallel() - s := newTestSite(t) - page, _ := s.NewPage(filepath.FromSlash("blue/file1.md")) - page.ReadFrom(strings.NewReader(simplePage)) - if page.Section() != "blue" { - t.Errorf("Section should be %s, got: %s", "blue", page.Section()) - } -} - -func TestSliceToLower(t *testing.T) { - t.Parallel() - tests := []struct { - value []string - expected []string - }{ - {[]string{"a", "b", "c"}, []string{"a", "b", "c"}}, - {[]string{"a", "B", "c"}, []string{"a", "b", "c"}}, - {[]string{"A", "B", "C"}, []string{"a", "b", "c"}}, - } - - for _, test := range tests { - res := helpers.SliceToLower(test.value) - for i, val := range res { - if val != test.expected[i] { - t.Errorf("Case mismatch. Expected %s, got %s", test.expected[i], res[i]) - } - } - } -} - func TestPagePaths(t *testing.T) { t.Parallel() @@ -1254,210 +912,11 @@ func TestPagePaths(t *testing.T) { writeSource(t, fs, filepath.Join("content", filepath.FromSlash(test.path)), test.content) s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - require.Len(t, s.RegularPages, 1) - - } -} + require.Len(t, s.RegularPages(), 1) -var pagesWithPublishedFalse = `--- -title: okay -published: false ---- -some content -` -var pageWithPublishedTrue = `--- -title: okay -published: true ---- -some content -` - -func TestPublishedFrontMatter(t *testing.T) { - t.Parallel() - s := newTestSite(t) - p, err := s.newPageFrom(strings.NewReader(pagesWithPublishedFalse), "content/post/broken.md") - if err != nil { - t.Fatalf("err during parse: %s", err) - } - if !p.Draft { - t.Errorf("expected true, got %t", p.Draft) - } - p, err = s.newPageFrom(strings.NewReader(pageWithPublishedTrue), "content/post/broken.md") - if err != nil { - t.Fatalf("err during parse: %s", err) - } - if p.Draft { - t.Errorf("expected false, got %t", p.Draft) } } -var pagesDraftTemplate = []string{`--- -title: "okay" -draft: %t ---- -some content -`, - `+++ -title = "okay" -draft = %t -+++ - -some content -`, -} - -func TestDraft(t *testing.T) { - t.Parallel() - s := newTestSite(t) - for _, draft := range []bool{true, false} { - for i, templ := range pagesDraftTemplate { - pageContent := fmt.Sprintf(templ, draft) - p, err := s.newPageFrom(strings.NewReader(pageContent), "content/post/broken.md") - if err != nil { - t.Fatalf("err during parse: %s", err) - } - if p.Draft != draft { - t.Errorf("[%d] expected %t, got %t", i, draft, p.Draft) - } - } - } -} - -var pagesParamsTemplate = []string{`+++ -title = "okay" -draft = false -tags = [ "hugo", "web" ] -social= [ - [ "a", "#" ], - [ "b", "#" ], -] -+++ -some content -`, - `--- -title: "okay" -draft: false -tags: - - hugo - - web -social: - - - a - - "#" - - - b - - "#" ---- -some content -`, - `{ - "title": "okay", - "draft": false, - "tags": [ "hugo", "web" ], - "social": [ - [ "a", "#" ], - [ "b", "#" ] - ] -} -some content -`, -} - -func TestPageParams(t *testing.T) { - t.Parallel() - s := newTestSite(t) - wantedMap := map[string]interface{}{ - "tags": []string{"hugo", "web"}, - // Issue #2752 - "social": []interface{}{ - []interface{}{"a", "#"}, - []interface{}{"b", "#"}, - }, - } - - for i, c := range pagesParamsTemplate { - p, err := s.newPageFrom(strings.NewReader(c), "content/post/params.md") - require.NoError(t, err, "err during parse", "#%d", i) - for key := range wantedMap { - assert.Equal(t, wantedMap[key], p.params[key], "#%d", key) - } - } -} - -func TestTraverse(t *testing.T) { - exampleParams := `--- -rating: "5 stars" -tags: - - hugo - - web -social: - twitter: "@jxxf" - facebook: "https://example.com" ----` - t.Parallel() - s := newTestSite(t) - p, _ := s.newPageFrom(strings.NewReader(exampleParams), "content/post/params.md") - - topLevelKeyValue, _ := p.Param("rating") - assert.Equal(t, "5 stars", topLevelKeyValue) - - nestedStringKeyValue, _ := p.Param("social.twitter") - assert.Equal(t, "@jxxf", nestedStringKeyValue) - - nonexistentKeyValue, _ := p.Param("doesn't.exist") - assert.Nil(t, nonexistentKeyValue) -} - -func TestPageSimpleMethods(t *testing.T) { - t.Parallel() - s := newTestSite(t) - for i, this := range []struct { - assertFunc func(p *Page) bool - }{ - {func(p *Page) bool { return !p.IsNode() }}, - {func(p *Page) bool { return p.IsPage() }}, - {func(p *Page) bool { return p.Plain() == "Do Be Do Be Do" }}, - {func(p *Page) bool { return strings.Join(p.PlainWords(), " ") == "Do Be Do Be Do" }}, - } { - - p, _ := s.NewPage("Test") - p.workContent = []byte("<h1>Do Be Do Be Do</h1>") - p.resetContent() - if !this.assertFunc(p) { - t.Errorf("[%d] Page method error", i) - } - } -} - -func TestIndexPageSimpleMethods(t *testing.T) { - s := newTestSite(t) - t.Parallel() - for i, this := range []struct { - assertFunc func(n *Page) bool - }{ - {func(n *Page) bool { return n.IsNode() }}, - {func(n *Page) bool { return !n.IsPage() }}, - {func(n *Page) bool { return n.Scratch() != nil }}, - {func(n *Page) bool { return n.Hugo().Version() != "" }}, - } { - - n := s.newHomePage() - - if !this.assertFunc(n) { - t.Errorf("[%d] Node method error", i) - } - } -} - -func TestKind(t *testing.T) { - t.Parallel() - // Add tests for these constants to make sure they don't change - require.Equal(t, "page", KindPage) - require.Equal(t, "home", KindHome) - require.Equal(t, "section", KindSection) - require.Equal(t, "taxonomy", KindTaxonomy) - require.Equal(t, "taxonomyTerm", KindTaxonomyTerm) - -} - func TestTranslationKey(t *testing.T) { t.Parallel() assert := require.New(t) @@ -1468,13 +927,13 @@ func TestTranslationKey(t *testing.T) { s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - require.Len(t, s.RegularPages, 2) + require.Len(t, s.RegularPages(), 2) home, _ := s.Info.Home() assert.NotNil(home) assert.Equal("home", home.TranslationKey()) - assert.Equal("page/k1", s.RegularPages[0].TranslationKey()) - p2 := s.RegularPages[1] + assert.Equal("page/k1", s.RegularPages()[0].TranslationKey()) + p2 := s.RegularPages()[1] assert.Equal("page/sect/simple", p2.TranslationKey()) @@ -1490,9 +949,9 @@ func TestChompBOM(t *testing.T) { s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - require.Len(t, s.RegularPages, 1) + require.Len(t, s.RegularPages(), 1) - p := s.RegularPages[0] + p := s.RegularPages()[0] checkPageTitle(t, p, "Simple") } @@ -1554,6 +1013,43 @@ but if you like it, hit :+1: and get subscribed! } +func TestPageHTMLContent(t *testing.T) { + b := newTestSitesBuilder(t) + b.WithSimpleConfigFile() + + frontmatter := `--- +title: "HTML Content" +--- +` + b.WithContent("regular.html", frontmatter+`<h1>Hugo</h1>`) + b.WithContent("noblackfridayforyou.html", frontmatter+`**Hugo!**`) + b.WithContent("manualsummary.html", frontmatter+` +<p>This is summary</p> +<!--more--> +<p>This is the main content.</p>`) + + b.Build(BuildCfg{}) + + b.AssertFileContent( + "public/regular/index.html", + "Single: HTML Content|Hello|en|RelPermalink: /regular/|", + "Summary: Hugo|Truncated: false") + + b.AssertFileContent( + "public/noblackfridayforyou/index.html", + "Permalink: http://example.com/noblackfridayforyou/|**Hugo!**|", + ) + + // https://github.com/gohugoio/hugo/issues/5723 + b.AssertFileContent( + "public/manualsummary/index.html", + "Single: HTML Content|Hello|en|RelPermalink: /manualsummary/|", + "Summary: \n<p>This is summary</p>\n|Truncated: true", + "|<p>This is the main content.</p>|", + ) + +} + // https://github.com/gohugoio/hugo/issues/5381 func TestPageManualSummary(t *testing.T) { b := newTestSitesBuilder(t) @@ -1670,17 +1166,6 @@ Content:{{ .Content }} } -// TODO(bep) this may be useful for other tests. -func compareObjects(a interface{}, b interface{}) bool { - aStr := strings.Split(fmt.Sprintf("%v", a), "") - sort.Strings(aStr) - - bStr := strings.Split(fmt.Sprintf("%v", b), "") - sort.Strings(bStr) - - return strings.Join(aStr, "") == strings.Join(bStr, "") -} - func TestShouldBuild(t *testing.T) { t.Parallel() var past = time.Date(2009, 11, 17, 20, 34, 58, 651387237, time.UTC) @@ -1773,7 +1258,7 @@ tags: s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - require.Len(t, s.RegularPages, 4) + require.Len(t, s.RegularPages(), 4) pathFunc := func(s string) string { if uglyURLs { @@ -1804,7 +1289,7 @@ tags: } - p := s.RegularPages[0] + p := s.RegularPages()[0] if uglyURLs { require.Equal(t, "/post/test0.dot.html", p.RelPermalink()) } else { @@ -1900,7 +1385,7 @@ Summary: In Chinese, 好 means good. b.CreateSites().Build(BuildCfg{}) assert.Equal(1, len(b.H.Sites)) - require.Len(t, b.H.Sites[0].RegularPages, 6) + require.Len(t, b.H.Sites[0].RegularPages(), 6) b.AssertFileContent("public/p1/index.html", "WordCount: 510\nFuzzyWordCount: 600\nReadingTime: 3\nLen Plain: 2550\nLen PlainWords: 510\nTruncated: false\nLen Summary: 2549\nLen Content: 2557") @@ -1939,15 +1424,3 @@ title: Scratch Me! b.AssertFileContent("public/index.html", "B: bv") b.AssertFileContent("public/scratchme/index.html", "C: cv") } - -func BenchmarkParsePage(b *testing.B) { - s := newTestSite(b) - f, _ := os.Open("testdata/redis.cn.md") - var buf bytes.Buffer - buf.ReadFrom(f) - b.ResetTimer() - for i := 0; i < b.N; i++ { - page, _ := s.NewPage("bench") - page.ReadFrom(bytes.NewReader(buf.Bytes())) - } -} diff --git a/hugolib/page_time_integration_test.go b/hugolib/page_time_integration_test.go deleted file mode 100644 index f180afa5e..000000000 --- a/hugolib/page_time_integration_test.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "os" - "strings" - "sync" - "testing" - "time" - - "github.com/spf13/cast" -) - -const ( - pageWithInvalidDate = `--- -date: 2010-05-02_15:29:31+08:00 ---- -Page With Invalid Date (replace T with _ for RFC 3339)` - - pageWithDateRFC3339 = `--- -date: 2010-05-02T15:29:31+08:00 ---- -Page With Date RFC3339` - - pageWithDateRFC3339NoT = `--- -date: 2010-05-02 15:29:31+08:00 ---- -Page With Date RFC3339_NO_T` - - pageWithRFC1123 = `--- -date: Sun, 02 May 2010 15:29:31 PST ---- -Page With Date RFC1123` - - pageWithDateRFC1123Z = `--- -date: Sun, 02 May 2010 15:29:31 +0800 ---- -Page With Date RFC1123Z` - - pageWithDateRFC822 = `--- -date: 02 May 10 15:29 PST ---- -Page With Date RFC822` - - pageWithDateRFC822Z = `--- -date: 02 May 10 15:29 +0800 ---- -Page With Date RFC822Z` - - pageWithDateANSIC = `--- -date: Sun May 2 15:29:31 2010 ---- -Page With Date ANSIC` - - pageWithDateUnixDate = `--- -date: Sun May 2 15:29:31 PST 2010 ---- -Page With Date UnixDate` - - pageWithDateRubyDate = `--- -date: Sun May 02 15:29:31 +0800 2010 ---- -Page With Date RubyDate` - - pageWithDateHugoYearNumeric = `--- -date: 2010-05-02 ---- -Page With Date HugoYearNumeric` - - pageWithDateHugoYear = `--- -date: 02 May 2010 ---- -Page With Date HugoYear` - - pageWithDateHugoLong = `--- -date: 02 May 2010 15:29 PST ---- -Page With Date HugoLong` -) - -func TestDegenerateDateFrontMatter(t *testing.T) { - t.Parallel() - s := newTestSite(t) - p, _ := s.newPageFrom(strings.NewReader(pageWithInvalidDate), "page/with/invalid/date") - if p.Date != *new(time.Time) { - t.Fatalf("Date should be set to time.Time zero value. Got: %s", p.Date) - } -} - -func TestParsingDateInFrontMatter(t *testing.T) { - t.Parallel() - s := newTestSite(t) - tests := []struct { - buf string - dt string - }{ - {pageWithDateRFC3339, "2010-05-02T15:29:31+08:00"}, - {pageWithDateRFC3339NoT, "2010-05-02T15:29:31+08:00"}, - {pageWithDateRFC1123Z, "2010-05-02T15:29:31+08:00"}, - {pageWithDateRFC822Z, "2010-05-02T15:29:00+08:00"}, - {pageWithDateANSIC, "2010-05-02T15:29:31Z"}, - {pageWithDateRubyDate, "2010-05-02T15:29:31+08:00"}, - {pageWithDateHugoYearNumeric, "2010-05-02T00:00:00Z"}, - {pageWithDateHugoYear, "2010-05-02T00:00:00Z"}, - } - - tzShortCodeTests := []struct { - buf string - dt string - }{ - {pageWithRFC1123, "2010-05-02T15:29:31-08:00"}, - {pageWithDateRFC822, "2010-05-02T15:29:00-08:00Z"}, - {pageWithDateUnixDate, "2010-05-02T15:29:31-08:00"}, - {pageWithDateHugoLong, "2010-05-02T15:21:00+08:00"}, - } - - if _, err := time.LoadLocation("PST"); err == nil { - tests = append(tests, tzShortCodeTests...) - } else { - fmt.Fprintf(os.Stderr, "Skipping shortname timezone tests.\n") - } - - for _, test := range tests { - dt, e := time.Parse(time.RFC3339, test.dt) - if e != nil { - t.Fatalf("Unable to parse date time (RFC3339) for running the test: %s", e) - } - p, err := s.newPageFrom(strings.NewReader(test.buf), "page/with/date") - if err != nil { - t.Fatalf("Expected to be able to parse page.") - } - if !dt.Equal(p.Date) { - t.Errorf("Date does not equal frontmatter:\n%s\nExpecting: %s\n Got: %s. Diff: %s\n internal: %#v\n %#v", test.buf, dt, p.Date, dt.Sub(p.Date), dt, p.Date) - } - } -} - -// Temp test https://github.com/gohugoio/hugo/issues/3059 -func TestParsingDateParallel(t *testing.T) { - t.Parallel() - - var wg sync.WaitGroup - - for j := 0; j < 100; j++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < 100; j++ { - dateStr := "2010-05-02 15:29:31 +08:00" - - dt, err := time.Parse("2006-01-02 15:04:05 -07:00", dateStr) - if err != nil { - t.Fatal(err) - } - - if dt.Year() != 2010 { - t.Fatal("time.Parse: Invalid date:", dt) - } - - dt2 := cast.ToTime(dateStr) - - if dt2.Year() != 2010 { - t.Fatal("cast.ToTime: Invalid date:", dt2.Year()) - } - } - }() - } - wg.Wait() - -} diff --git a/hugolib/page_unwrap.go b/hugolib/page_unwrap.go new file mode 100644 index 000000000..eda6636d1 --- /dev/null +++ b/hugolib/page_unwrap.go @@ -0,0 +1,50 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/resources/page" +) + +// Wraps a Page. +type pageWrapper interface { + page() page.Page +} + +// unwrapPage is used in equality checks and similar. +func unwrapPage(in interface{}) (page.Page, error) { + switch v := in.(type) { + case *pageState: + return v, nil + case pageWrapper: + return v.page(), nil + case page.Page: + return v, nil + case nil: + return nil, nil + default: + return nil, errors.Errorf("unwrapPage: %T not supported", in) + } +} + +func mustUnwrapPage(in interface{}) page.Page { + p, err := unwrapPage(in) + if err != nil { + panic(err) + } + + return p +} diff --git a/hugolib/page_resource.go b/hugolib/page_unwrap_test.go index 201076e8b..23747dce8 100644 --- a/hugolib/page_resource.go +++ b/hugolib/page_unwrap_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,10 +14,24 @@ package hugolib import ( - "github.com/gohugoio/hugo/resources/resource" -) + "testing" -var ( - _ resource.Resource = (*Page)(nil) - _ resource.Resource = (*PageOutput)(nil) + "github.com/gohugoio/hugo/resources/page" + "github.com/stretchr/testify/require" ) + +func TestUnwrapPage(t *testing.T) { + assert := require.New(t) + + p := &pageState{} + + assert.Equal(p, mustUnwrap(newPageForShortcode(p))) +} + +func mustUnwrap(v interface{}) page.Page { + p, err := unwrapPage(v) + if err != nil { + panic(err) + } + return p +} diff --git a/hugolib/page_without_content.go b/hugolib/page_without_content.go deleted file mode 100644 index 3659efaea..000000000 --- a/hugolib/page_without_content.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "html/template" -) - -// PageWithoutContent is sent to the shortcodes. They cannot access the content -// they're a part of. It would cause an infinite regress. -// -// Go doesn't support virtual methods, so this careful dance is currently (I think) -// the best we can do. -type PageWithoutContent struct { - *Page -} - -// Content returns an empty string. -func (p *PageWithoutContent) Content() (interface{}, error) { - return "", nil -} - -// Truncated always returns false. -func (p *PageWithoutContent) Truncated() bool { - return false -} - -// Summary returns an empty string. -func (p *PageWithoutContent) Summary() template.HTML { - return "" -} - -// WordCount always returns 0. -func (p *PageWithoutContent) WordCount() int { - return 0 -} - -// ReadingTime always returns 0. -func (p *PageWithoutContent) ReadingTime() int { - return 0 -} - -// FuzzyWordCount always returns 0. -func (p *PageWithoutContent) FuzzyWordCount() int { - return 0 -} - -// Plain returns an empty string. -func (p *PageWithoutContent) Plain() string { - return "" -} - -// PlainWords returns an empty string slice. -func (p *PageWithoutContent) PlainWords() []string { - return []string{} -} diff --git a/hugolib/pagebundler.go b/hugolib/pagebundler.go index 62ef2b52b..546b125ff 100644 --- a/hugolib/pagebundler.go +++ b/hugolib/pagebundler.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ import ( "context" "fmt" "math" + "path/filepath" "runtime" _errors "github.com/pkg/errors" @@ -38,12 +39,12 @@ type siteContentProcessor struct { fileSinglesChan chan *fileInfo // These assets should be just copied to destination. - fileAssetsChan chan []pathLangFile + fileAssetsChan chan pathLangFile numWorkers int // The output Pages - pagesChan chan *Page + pagesChan chan *pageState // Used for partial rebuilds (aka. live reload) // Will signal replacement of pages in the site collection. @@ -64,9 +65,9 @@ func (s *siteContentProcessor) processSingle(fi *fileInfo) { } } -func (s *siteContentProcessor) processAssets(assets []pathLangFile) { +func (s *siteContentProcessor) processAsset(asset pathLangFile) { select { - case s.fileAssetsChan <- assets: + case s.fileAssetsChan <- asset: case <-s.ctx.Done(): } } @@ -77,7 +78,7 @@ func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *s numWorkers = n } - numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.owner.Sites)))) + numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.h.Sites)))) return &siteContentProcessor{ ctx: ctx, @@ -86,9 +87,9 @@ func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *s handleContent: newHandlerChain(s), fileBundlesChan: make(chan *bundleDir, numWorkers), fileSinglesChan: make(chan *fileInfo, numWorkers), - fileAssetsChan: make(chan []pathLangFile, numWorkers), + fileAssetsChan: make(chan pathLangFile, numWorkers), numWorkers: numWorkers, - pagesChan: make(chan *Page, numWorkers), + pagesChan: make(chan *pageState, numWorkers), } } @@ -127,6 +128,7 @@ func (s *siteContentProcessor) process(ctx context.Context) error { if !ok { return nil } + err := s.readAndConvertContentFile(f) if err != nil { return err @@ -140,22 +142,20 @@ func (s *siteContentProcessor) process(ctx context.Context) error { g2.Go(func() error { for { select { - case files, ok := <-s.fileAssetsChan: + case file, ok := <-s.fileAssetsChan: if !ok { return nil } - for _, file := range files { - f, err := s.site.BaseFs.Content.Fs.Open(file.Filename()) - if err != nil { - return _errors.Wrap(err, "failed to open assets file") - } - err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, file.Path(), f) - f.Close() - if err != nil { - return err - } + f, err := s.site.BaseFs.Content.Fs.Open(file.Filename()) + if err != nil { + return _errors.Wrap(err, "failed to open assets file") + } + filename := filepath.Join(s.site.GetTargetLanguageBasePath(), file.Path()) + err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, filename, f) + f.Close() + if err != nil { + return err } - case <-ctx.Done(): return ctx.Err() } @@ -192,8 +192,6 @@ func (s *siteContentProcessor) process(ctx context.Context) error { return err } - s.site.rawAllPages.sort() - return nil } diff --git a/hugolib/pagebundler_capture.go b/hugolib/pagebundler_capture.go index 446d3b0c7..17a4b865a 100644 --- a/hugolib/pagebundler_capture.go +++ b/hugolib/pagebundler_capture.go @@ -116,7 +116,7 @@ func newCapturer( // these channels. type captureResultHandler interface { handleSingles(fis ...*fileInfo) - handleCopyFiles(fis ...pathLangFile) + handleCopyFile(fi pathLangFile) captureBundlesHandler } @@ -141,10 +141,10 @@ func (c *captureResultHandlerChain) handleBundles(b *bundleDirs) { } } -func (c *captureResultHandlerChain) handleCopyFiles(files ...pathLangFile) { +func (c *captureResultHandlerChain) handleCopyFile(file pathLangFile) { for _, h := range c.handlers { if hh, ok := h.(captureResultHandler); ok { - hh.handleCopyFiles(files...) + hh.handleCopyFile(file) } } } @@ -444,7 +444,7 @@ func (c *capturer) handleNonBundle( } c.handler.handleSingles(f) } else { - c.handler.handleCopyFiles(fi) + c.handler.handleCopyFile(fi) } } } @@ -457,7 +457,7 @@ func (c *capturer) copyOrHandleSingle(fi *fileInfo) { c.handler.handleSingles(fi) } else { // These do not currently need any further processing. - c.handler.handleCopyFiles(fi) + c.handler.handleCopyFile(fi) } } diff --git a/hugolib/pagebundler_capture_test.go b/hugolib/pagebundler_capture_test.go index d6128352c..b6d9822af 100644 --- a/hugolib/pagebundler_capture_test.go +++ b/hugolib/pagebundler_capture_test.go @@ -64,12 +64,10 @@ func (s *storeFilenames) handleBundles(d *bundleDirs) { s.dirKeys = append(s.dirKeys, keys...) } -func (s *storeFilenames) handleCopyFiles(files ...pathLangFile) { +func (s *storeFilenames) handleCopyFile(file pathLangFile) { s.Lock() defer s.Unlock() - for _, file := range files { - s.copyNames = append(s.copyNames, filepath.ToSlash(file.Filename())) - } + s.copyNames = append(s.copyNames, filepath.ToSlash(file.Filename())) } func (s *storeFilenames) sortedStr() string { @@ -224,9 +222,9 @@ C: type noOpFileStore int -func (noOpFileStore) handleSingles(fis ...*fileInfo) {} -func (noOpFileStore) handleBundles(b *bundleDirs) {} -func (noOpFileStore) handleCopyFiles(files ...pathLangFile) {} +func (noOpFileStore) handleSingles(fis ...*fileInfo) {} +func (noOpFileStore) handleBundles(b *bundleDirs) {} +func (noOpFileStore) handleCopyFile(file pathLangFile) {} func BenchmarkPageBundlerCapture(b *testing.B) { capturers := make([]*capturer, b.N) diff --git a/hugolib/pagebundler_handlers.go b/hugolib/pagebundler_handlers.go index 2df1f8765..c217b5e09 100644 --- a/hugolib/pagebundler_handlers.go +++ b/hugolib/pagebundler_handlers.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,11 +17,11 @@ import ( "errors" "fmt" "path/filepath" - "sort" + + "github.com/gohugoio/hugo/common/hugio" "strings" - "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" ) @@ -50,13 +50,9 @@ func init() { func newHandlerChain(s *Site) contentHandler { c := &contentHandlers{s: s} - contentFlow := c.parsePage(c.processFirstMatch( - // Handles all files with a content file extension. See above. + contentFlow := c.parsePage( c.handlePageContent(), - - // Every HTML file without front matter will be passed on to this handler. - c.handleHTMLContent(), - )) + ) c.rootHandler = c.processFirstMatch( contentFlow, @@ -93,12 +89,12 @@ func (c *contentHandlers) processFirstMatch(handlers ...contentHandler) func(ctx type handlerContext struct { // These are the pages stored in Site. - pages chan<- *Page + pages chan<- *pageState doNotAddToSiteCollections bool - currentPage *Page - parentPage *Page + currentPage *pageState + parentPage *pageState bundle *bundleDir @@ -110,10 +106,7 @@ type handlerContext struct { func (c *handlerContext) ext() string { if c.currentPage != nil { - if c.currentPage.Markup != "" { - return c.currentPage.Markup - } - return c.currentPage.Ext() + return c.currentPage.contentMarkupType() } if c.bundle != nil { @@ -175,9 +168,9 @@ func (c *handlerContext) isContentFile() bool { type ( handlerResult struct { - err error - handled bool - resource resource.Resource + err error + handled bool + result interface{} } contentHandler func(ctx *handlerContext) handlerResult @@ -196,27 +189,27 @@ func (c *contentHandlers) parsePage(h contentHandler) contentHandler { result := handlerResult{handled: true} fi := ctx.file() - f, err := fi.Open() - if err != nil { - return handlerResult{err: fmt.Errorf("(%s) failed to open content file: %s", fi.Filename(), err)} + content := func() (hugio.ReadSeekCloser, error) { + f, err := fi.Open() + if err != nil { + return nil, fmt.Errorf("failed to open content file %q: %s", fi.Filename(), err) + } + return f, nil } - defer f.Close() - - p := c.s.newPageFromFile(fi) - _, err = p.ReadFrom(f) + ps, err := newPageWithContent(fi, c.s, content) if err != nil { return handlerResult{err: err} } - if !p.shouldBuild() { + if !c.s.shouldBuild(ps) { if !ctx.doNotAddToSiteCollections { - ctx.pages <- p + ctx.pages <- ps } return result } - ctx.currentPage = p + ctx.currentPage = ps if ctx.bundle != nil { // Add the bundled files @@ -226,39 +219,20 @@ func (c *contentHandlers) parsePage(h contentHandler) contentHandler { if res.err != nil { return res } - if res.resource != nil { - if pageResource, ok := res.resource.(*Page); ok { - pageResource.resourcePath = filepath.ToSlash(childCtx.target) - pageResource.parent = p + if res.result != nil { + switch resv := res.result.(type) { + case *pageState: + resv.m.resourcePath = filepath.ToSlash(childCtx.target) + resv.parent = ps + ps.addResources(resv) + case resource.Resource: + ps.addResources(resv) + + default: + panic("Unknown type") } - p.Resources = append(p.Resources, res.resource) - } - } - - sort.SliceStable(p.Resources, func(i, j int) bool { - if p.Resources[i].ResourceType() < p.Resources[j].ResourceType() { - return true - } - - p1, ok1 := p.Resources[i].(*Page) - p2, ok2 := p.Resources[j].(*Page) - - if ok1 != ok2 { - return ok2 - } - - if ok1 { - return defaultPageSort(p1, p2) } - - return p.Resources[i].RelPermalink() < p.Resources[j].RelPermalink() - }) - - // Assign metadata from front matter if set - if len(p.resourcesMetadata) > 0 { - resources.AssignMetadata(p.resourcesMetadata, p.Resources...) } - } return h(ctx) @@ -267,58 +241,47 @@ func (c *contentHandlers) parsePage(h contentHandler) contentHandler { func (c *contentHandlers) handlePageContent() contentHandler { return func(ctx *handlerContext) handlerResult { - if ctx.supports("html", "htm") { - return notHandled - } - p := ctx.currentPage - p.workContent = p.renderContent(p.workContent) - - tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.workContent) - p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents) - p.workContent = tmpContent - if !ctx.doNotAddToSiteCollections { ctx.pages <- p } - return handlerResult{handled: true, resource: p} + return handlerResult{handled: true, result: p} } } -func (c *contentHandlers) handleHTMLContent() contentHandler { +func (c *contentHandlers) createResource() contentHandler { return func(ctx *handlerContext) handlerResult { - if !ctx.supports("html", "htm") { + if ctx.parentPage == nil { return notHandled } - p := ctx.currentPage - - if !ctx.doNotAddToSiteCollections { - ctx.pages <- p - } - - return handlerResult{handled: true, resource: p} - } -} + // TODO(bep) consolidate with multihost logic + clean up + outputFormats := ctx.parentPage.m.outputFormats() + seen := make(map[string]bool) + var targetBasePaths []string + // Make sure bundled resources are published to all of the ouptput formats' + // sub paths. + for _, f := range outputFormats { + p := f.Path + if seen[p] { + continue + } + seen[p] = true + targetBasePaths = append(targetBasePaths, p) -func (c *contentHandlers) createResource() contentHandler { - return func(ctx *handlerContext) handlerResult { - if ctx.parentPage == nil { - return notHandled } resource, err := c.s.ResourceSpec.New( resources.ResourceSourceDescriptor{ - TargetPathBuilder: ctx.parentPage.subResourceTargetPathFactory, + TargetPaths: ctx.parentPage.getTargetPaths, SourceFile: ctx.source, RelTargetFilename: ctx.target, - URLBase: c.s.GetURLLanguageBasePath(), - TargetBasePaths: []string{c.s.GetTargetLanguageBasePath()}, + TargetBasePaths: targetBasePaths, }) - return handlerResult{err: err, handled: true, resource: resource} + return handlerResult{err: err, handled: true, result: resource} } } diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go index ab0472059..870ea3de9 100644 --- a/hugolib/pagebundler_test.go +++ b/hugolib/pagebundler_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,12 +14,15 @@ package hugolib import ( - "github.com/gohugoio/hugo/common/loggers" - "os" + "path" "runtime" + "strings" "testing" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/helpers" "io" @@ -47,7 +50,11 @@ func TestPageBundlerSiteRegular(t *testing.T) { for _, baseURLPath := range []string{"", "/hugo"} { for _, canonify := range []bool{false, true} { for _, ugly := range []bool{false, true} { - t.Run(fmt.Sprintf("ugly=%t,canonify=%t,path=%s", ugly, canonify, baseURLPath), + baseURLPathId := baseURLPath + if baseURLPathId == "" { + baseURLPathId = "NONE" + } + t.Run(fmt.Sprintf("ugly=%t,canonify=%t,path=%s", ugly, canonify, baseURLPathId), func(t *testing.T) { baseURL := baseBaseURL + baseURLPath relURLBase := baseURLPath @@ -70,9 +77,10 @@ func TestPageBundlerSiteRegular(t *testing.T) { cfg.Set("outputFormats", map[string]interface{}{ "CUSTOMO": map[string]interface{}{ - "mediaType": media.HTMLType, - "baseName": "cindex", - "path": "cpath", + "mediaType": media.HTMLType, + "baseName": "cindex", + "path": "cpath", + "permalinkable": true, }, }) @@ -84,70 +92,92 @@ func TestPageBundlerSiteRegular(t *testing.T) { cfg.Set("uglyURLs", ugly) - s := buildSingleSite(t, deps.DepsCfg{Logger: loggers.NewWarningLogger(), Fs: fs, Cfg: cfg}, BuildCfg{}) + s := buildSingleSite(t, deps.DepsCfg{Logger: loggers.NewErrorLogger(), Fs: fs, Cfg: cfg}, BuildCfg{}) th := testHelper{s.Cfg, s.Fs, t} - assert.Len(s.RegularPages, 8) + assert.Len(s.RegularPages(), 8) - singlePage := s.getPage(KindPage, "a/1.md") + singlePage := s.getPage(page.KindPage, "a/1.md") assert.Equal("", singlePage.BundleType()) assert.NotNil(singlePage) assert.Equal(singlePage, s.getPage("page", "a/1")) assert.Equal(singlePage, s.getPage("page", "1")) - assert.Contains(singlePage.content(), "TheContent") + assert.Contains(content(singlePage), "TheContent") - if ugly { - assert.Equal(relURLBase+"/a/1.html", singlePage.RelPermalink()) - th.assertFileContent(filepath.FromSlash("/work/public/a/1.html"), "TheContent") + relFilename := func(basePath, outBase string) (string, string) { + rel := basePath + if ugly { + rel = strings.TrimSuffix(basePath, "/") + ".html" + } - } else { - assert.Equal(relURLBase+"/a/1/", singlePage.RelPermalink()) - th.assertFileContent(filepath.FromSlash("/work/public/a/1/index.html"), "TheContent") + var filename string + if !ugly { + filename = path.Join(basePath, outBase) + } else { + filename = rel + } + + rel = fmt.Sprintf("%s%s", relURLBase, rel) + + return rel, filename } + // Check both output formats + rel, filename := relFilename("/a/1/", "index.html") + th.assertFileContent(filepath.Join("/work/public", filename), + "TheContent", + "Single RelPermalink: "+rel, + ) + + rel, filename = relFilename("/cpath/a/1/", "cindex.html") + + th.assertFileContent(filepath.Join("/work/public", filename), + "TheContent", + "Single RelPermalink: "+rel, + ) + th.assertFileContent(filepath.FromSlash("/work/public/images/hugo-logo.png"), "content") // This should be just copied to destination. th.assertFileContent(filepath.FromSlash("/work/public/assets/pic1.png"), "content") - leafBundle1 := s.getPage(KindPage, "b/my-bundle/index.md") + leafBundle1 := s.getPage(page.KindPage, "b/my-bundle/index.md") assert.NotNil(leafBundle1) assert.Equal("leaf", leafBundle1.BundleType()) assert.Equal("b", leafBundle1.Section()) - sectionB := s.getPage(KindSection, "b") + sectionB := s.getPage(page.KindSection, "b") assert.NotNil(sectionB) home, _ := s.Info.Home() assert.Equal("branch", home.BundleType()) // This is a root bundle and should live in the "home section" // See https://github.com/gohugoio/hugo/issues/4332 - rootBundle := s.getPage(KindPage, "root") + rootBundle := s.getPage(page.KindPage, "root") assert.NotNil(rootBundle) assert.True(rootBundle.Parent().IsHome()) - if ugly { - assert.Equal(relURLBase+"/root.html", rootBundle.RelPermalink()) - } else { - assert.Equal(relURLBase+"/root/", rootBundle.RelPermalink()) + if !ugly { + th.assertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single RelPermalink: "+relURLBase+"/root/") + th.assertFileContent(filepath.FromSlash("/work/public/cpath/root/cindex.html"), "Single RelPermalink: "+relURLBase+"/cpath/root/") } - leafBundle2 := s.getPage(KindPage, "a/b/index.md") + leafBundle2 := s.getPage(page.KindPage, "a/b/index.md") assert.NotNil(leafBundle2) - unicodeBundle := s.getPage(KindPage, "c/bundle/index.md") + unicodeBundle := s.getPage(page.KindPage, "c/bundle/index.md") assert.NotNil(unicodeBundle) - pageResources := leafBundle1.Resources.ByType(pageResourceType) + pageResources := leafBundle1.Resources().ByType(pageResourceType) assert.Len(pageResources, 2) - firstPage := pageResources[0].(*Page) - secondPage := pageResources[1].(*Page) - assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/1.md"), firstPage.pathOrTitle(), secondPage.pathOrTitle()) - assert.Contains(firstPage.content(), "TheContent") - assert.Equal(6, len(leafBundle1.Resources)) + firstPage := pageResources[0].(page.Page) + secondPage := pageResources[1].(page.Page) + assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/1.md"), firstPage.File().Filename(), secondPage.File().Filename()) + assert.Contains(content(firstPage), "TheContent") + assert.Equal(6, len(leafBundle1.Resources())) // Verify shortcode in bundled page - assert.Contains(secondPage.content(), filepath.FromSlash("MyShort in b/my-bundle/2.md")) + assert.Contains(content(secondPage), filepath.FromSlash("MyShort in b/my-bundle/2.md")) // https://github.com/gohugoio/hugo/issues/4582 assert.Equal(leafBundle1, firstPage.Parent()) @@ -157,20 +187,10 @@ func TestPageBundlerSiteRegular(t *testing.T) { assert.Equal(secondPage, pageResources.GetMatch("2*")) assert.Nil(pageResources.GetMatch("doesnotexist*")) - imageResources := leafBundle1.Resources.ByType("image") + imageResources := leafBundle1.Resources().ByType("image") assert.Equal(3, len(imageResources)) - image := imageResources[0] - - altFormat := leafBundle1.OutputFormats().Get("CUSTOMO") - assert.NotNil(altFormat) - - assert.Equal(baseURL+"/2017/pageslug/c/logo.png", image.Permalink()) - th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") - th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content") - - // Custom media type defined in site config. - assert.Len(leafBundle1.Resources.ByType("bepsays"), 1) + assert.NotNil(leafBundle1.OutputFormats().Get("CUSTOMO")) relPermalinker := func(s string) string { return fmt.Sprintf(s, relURLBase) @@ -180,12 +200,33 @@ func TestPageBundlerSiteRegular(t *testing.T) { return fmt.Sprintf(s, baseURL) } - if permalinker == nil { + if ugly { + th.assertFileContent("/work/public/2017/pageslug.html", + relPermalinker("Single RelPermalink: %s/2017/pageslug.html"), + permalinker("Single Permalink: %s/2017/pageslug.html"), + relPermalinker("Sunset RelPermalink: %s/2017/pageslug/sunset1.jpg"), + permalinker("Sunset Permalink: %s/2017/pageslug/sunset1.jpg")) + } else { + th.assertFileContent("/work/public/2017/pageslug/index.html", + relPermalinker("Sunset RelPermalink: %s/2017/pageslug/sunset1.jpg"), + permalinker("Sunset Permalink: %s/2017/pageslug/sunset1.jpg")) + + th.assertFileContent("/work/public/cpath/2017/pageslug/cindex.html", + relPermalinker("Single RelPermalink: %s/cpath/2017/pageslug/"), + relPermalinker("Short Sunset RelPermalink: %s/cpath/2017/pageslug/sunset2.jpg"), + relPermalinker("Sunset RelPermalink: %s/cpath/2017/pageslug/sunset1.jpg"), + permalinker("Sunset Permalink: %s/cpath/2017/pageslug/sunset1.jpg"), + ) } + th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") + th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content") + th.assertFileNotExist("/work/public/cpath/cpath/2017/pageslug/c/logo.png") + + // Custom media type defined in site config. + assert.Len(leafBundle1.Resources().ByType("bepsays"), 1) + if ugly { - assert.Equal(relURLBase+"/2017/pageslug.html", leafBundle1.RelPermalink()) - assert.Equal(baseURL+"/2017/pageslug.html", leafBundle1.Permalink()) th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug.html"), "TheContent", relPermalinker("Sunset RelPermalink: %s/2017/pageslug/sunset1.jpg"), @@ -202,23 +243,15 @@ func TestPageBundlerSiteRegular(t *testing.T) { ) th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug.html"), "TheContent") - assert.Equal(relURLBase+"/a/b.html", leafBundle2.RelPermalink()) - // 은행 - assert.Equal(relURLBase+"/c/%EC%9D%80%ED%96%89.html", unicodeBundle.RelPermalink()) - th.assertFileContent(filepath.FromSlash("/work/public/c/은행.html"), "Content for 은행") th.assertFileContent(filepath.FromSlash("/work/public/c/은행/logo-은행.png"), "은행 PNG") } else { - assert.Equal(relURLBase+"/2017/pageslug/", leafBundle1.RelPermalink()) - assert.Equal(baseURL+"/2017/pageslug/", leafBundle1.Permalink()) th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "TheContent") th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/cindex.html"), "TheContent") th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/index.html"), "Single Title") th.assertFileContent(filepath.FromSlash("/work/public/root/index.html"), "Single Title") - assert.Equal(relURLBase+"/a/b/", leafBundle2.RelPermalink()) - } }) @@ -249,11 +282,11 @@ func TestPageBundlerSiteMultilingual(t *testing.T) { s := sites.Sites[0] - assert.Equal(8, len(s.RegularPages)) - assert.Equal(16, len(s.Pages)) - assert.Equal(31, len(s.AllPages)) + assert.Equal(8, len(s.RegularPages())) + assert.Equal(16, len(s.Pages())) + assert.Equal(31, len(s.AllPages())) - bundleWithSubPath := s.getPage(KindPage, "lb/index") + bundleWithSubPath := s.getPage(page.KindPage, "lb/index") assert.NotNil(bundleWithSubPath) // See https://github.com/gohugoio/hugo/issues/4312 @@ -267,30 +300,30 @@ func TestPageBundlerSiteMultilingual(t *testing.T) { // and probably also just b (aka "my-bundle") // These may also be translated, so we also need to test that. // "bf", "my-bf-bundle", "index.md + nn - bfBundle := s.getPage(KindPage, "bf/my-bf-bundle/index") + bfBundle := s.getPage(page.KindPage, "bf/my-bf-bundle/index") assert.NotNil(bfBundle) - assert.Equal("en", bfBundle.Lang()) - assert.Equal(bfBundle, s.getPage(KindPage, "bf/my-bf-bundle/index.md")) - assert.Equal(bfBundle, s.getPage(KindPage, "bf/my-bf-bundle")) - assert.Equal(bfBundle, s.getPage(KindPage, "my-bf-bundle")) + assert.Equal("en", bfBundle.Language().Lang) + assert.Equal(bfBundle, s.getPage(page.KindPage, "bf/my-bf-bundle/index.md")) + assert.Equal(bfBundle, s.getPage(page.KindPage, "bf/my-bf-bundle")) + assert.Equal(bfBundle, s.getPage(page.KindPage, "my-bf-bundle")) nnSite := sites.Sites[1] - assert.Equal(7, len(nnSite.RegularPages)) + assert.Equal(7, len(nnSite.RegularPages())) - bfBundleNN := nnSite.getPage(KindPage, "bf/my-bf-bundle/index") + bfBundleNN := nnSite.getPage(page.KindPage, "bf/my-bf-bundle/index") assert.NotNil(bfBundleNN) - assert.Equal("nn", bfBundleNN.Lang()) - assert.Equal(bfBundleNN, nnSite.getPage(KindPage, "bf/my-bf-bundle/index.nn.md")) - assert.Equal(bfBundleNN, nnSite.getPage(KindPage, "bf/my-bf-bundle")) - assert.Equal(bfBundleNN, nnSite.getPage(KindPage, "my-bf-bundle")) + assert.Equal("nn", bfBundleNN.Language().Lang) + assert.Equal(bfBundleNN, nnSite.getPage(page.KindPage, "bf/my-bf-bundle/index.nn.md")) + assert.Equal(bfBundleNN, nnSite.getPage(page.KindPage, "bf/my-bf-bundle")) + assert.Equal(bfBundleNN, nnSite.getPage(page.KindPage, "my-bf-bundle")) // See https://github.com/gohugoio/hugo/issues/4295 // Every resource should have its Name prefixed with its base folder. - cBundleResources := bundleWithSubPath.Resources.Match("c/**") + cBundleResources := bundleWithSubPath.Resources().Match("c/**") assert.Equal(4, len(cBundleResources)) - bundlePage := bundleWithSubPath.Resources.GetMatch("c/page*") + bundlePage := bundleWithSubPath.Resources().GetMatch("c/page*") assert.NotNil(bundlePage) - assert.IsType(&Page{}, bundlePage) + assert.IsType(&pageState{}, bundlePage) }) } @@ -329,15 +362,15 @@ func TestMultilingualDisableLanguage(t *testing.T) { s := sites.Sites[0] - assert.Equal(8, len(s.RegularPages)) - assert.Equal(16, len(s.Pages)) + assert.Equal(8, len(s.RegularPages())) + assert.Equal(16, len(s.Pages())) // No nn pages - assert.Equal(16, len(s.AllPages)) + assert.Equal(16, len(s.AllPages())) for _, p := range s.rawAllPages { - assert.True(p.Lang() != "nn") + assert.True(p.Language().Lang != "nn") } - for _, p := range s.AllPages { - assert.True(p.Lang() != "nn") + for _, p := range s.AllPages() { + assert.True(p.Language().Lang != "nn") } } @@ -358,11 +391,11 @@ func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) { th := testHelper{s.Cfg, s.Fs, t} - assert.Equal(7, len(s.RegularPages)) - a1Bundle := s.getPage(KindPage, "symbolic2/a1/index.md") + assert.Equal(7, len(s.RegularPages())) + a1Bundle := s.getPage(page.KindPage, "symbolic2/a1/index.md") assert.NotNil(a1Bundle) - assert.Equal(2, len(a1Bundle.Resources)) - assert.Equal(1, len(a1Bundle.Resources.ByType(pageResourceType))) + assert.Equal(2, len(a1Bundle.Resources())) + assert.Equal(1, len(a1Bundle.Resources().ByType(pageResourceType))) th.assertFileContent(filepath.FromSlash(workDir+"/public/a/page/index.html"), "TheContent") th.assertFileContent(filepath.FromSlash(workDir+"/public/symbolic1/s1/index.html"), "TheContent") @@ -416,28 +449,27 @@ HEADLESS {{< myShort >}} s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - assert.Equal(1, len(s.RegularPages)) + assert.Equal(1, len(s.RegularPages())) assert.Equal(1, len(s.headlessPages)) - regular := s.getPage(KindPage, "a/index") + regular := s.getPage(page.KindPage, "a/index") assert.Equal("/a/s1/", regular.RelPermalink()) - headless := s.getPage(KindPage, "b/index") + headless := s.getPage(page.KindPage, "b/index") assert.NotNil(headless) - assert.True(headless.headless) assert.Equal("Headless Bundle in Topless Bar", headless.Title()) assert.Equal("", headless.RelPermalink()) assert.Equal("", headless.Permalink()) - assert.Contains(headless.content(), "HEADLESS SHORTCODE") + assert.Contains(content(headless), "HEADLESS SHORTCODE") - headlessResources := headless.Resources + headlessResources := headless.Resources() assert.Equal(3, len(headlessResources)) assert.Equal(2, len(headlessResources.Match("l*"))) pageResource := headlessResources.GetMatch("p*") assert.NotNil(pageResource) - assert.IsType(&Page{}, pageResource) - p := pageResource.(*Page) - assert.Contains(p.content(), "SHORTCODE") + assert.IsType(&pageState{}, pageResource) + p := pageResource.(page.Page) + assert.Contains(content(p), "SHORTCODE") assert.Equal("p1.md", p.Name()) th := testHelper{s.Cfg, s.Fs, t} @@ -451,6 +483,91 @@ HEADLESS {{< myShort >}} } +func TestMultiSiteBundles(t *testing.T) { + assert := require.New(t) + b := newTestSitesBuilder(t) + b.WithConfigFile("toml", ` + +baseURL = "http://example.com/" + +defaultContentLanguage = "en" + +[languages] +[languages.en] +weight = 10 +contentDir = "content/en" +[languages.nn] +weight = 20 +contentDir = "content/nn" + + +`) + + b.WithContent("en/mybundle/index.md", ` +--- +headless: true +--- + +`) + + b.WithContent("nn/mybundle/index.md", ` +--- +headless: true +--- + +`) + + b.WithContent("en/mybundle/data.yaml", `data en`) + b.WithContent("en/mybundle/forms.yaml", `forms en`) + b.WithContent("nn/mybundle/data.yaml", `data nn`) + + b.WithContent("en/_index.md", ` +--- +Title: Home +--- + +Home content. + +`) + + b.WithContent("en/section-not-bundle/_index.md", ` +--- +Title: Section Page +--- + +Section content. + +`) + + b.WithContent("en/section-not-bundle/single.md", ` +--- +Title: Section Single +Date: 2018-02-01 +--- + +Single content. + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/nn/mybundle/data.yaml", "data nn") + b.AssertFileContent("public/nn/mybundle/forms.yaml", "forms en") + b.AssertFileContent("public/mybundle/data.yaml", "data en") + b.AssertFileContent("public/mybundle/forms.yaml", "forms en") + + assert.False(b.CheckExists("public/nn/nn/mybundle/data.yaml")) + assert.False(b.CheckExists("public/en/mybundle/data.yaml")) + + homeEn := b.H.Sites[0].home + assert.NotNil(homeEn) + assert.Equal(2018, homeEn.Date().Year()) + + b.AssertFileContent("public/section-not-bundle/index.html", "Section Page", "Content: <p>Section content.</p>") + b.AssertFileContent("public/section-not-bundle/single/index.html", "Section Single", "|<p>Single content.</p>") + +} + func newTestBundleSources(t *testing.T) (*hugofs.Fs, *viper.Viper) { cfg, fs := newTestCfg() assert := require.New(t) @@ -512,6 +629,8 @@ TheContent. singleLayout := ` Single Title: {{ .Title }} +Single RelPermalink: {{ .RelPermalink }} +Single Permalink: {{ .Permalink }} Content: {{ .Content }} {{ $sunset := .Resources.GetMatch "my-sunset-1*" }} {{ with $sunset }} @@ -532,7 +651,7 @@ Thumb RelPermalink: {{ $thumb.RelPermalink }} ` myShort := ` -MyShort in {{ .Page.Path }}: +MyShort in {{ .Page.File.Path }}: {{ $sunset := .Page.Resources.GetMatch "my-sunset-2*" }} {{ with $sunset }} Short Sunset RelPermalink: {{ .RelPermalink }} @@ -599,6 +718,7 @@ Content for 은행. assert.NoError(err) _, err = io.Copy(out, src) + assert.NoError(err) out.Close() src.Seek(0, 0) _, err = io.Copy(out2, src) diff --git a/hugolib/pagecollections.go b/hugolib/pagecollections.go index 78325344b..f62ea0905 100644 --- a/hugolib/pagecollections.go +++ b/hugolib/pagecollections.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,43 +18,65 @@ import ( "path" "path/filepath" "strings" + "sync" + + "github.com/pkg/errors" "github.com/gohugoio/hugo/cache" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/resources/page" ) +// Used in the page cache to mark more than one hit for a given key. +var ambiguityFlag = &pageState{} + // PageCollections contains the page collections for a site. type PageCollections struct { - // Includes only pages of all types, and only pages in the current language. - Pages Pages - // Includes all pages in all languages, including the current one. - // Includes pages of all types. - AllPages Pages + // Includes absolute all pages (of all types), including drafts etc. + rawAllPages pageStatePages - // A convenience cache for the traditional index types, taxonomies, home page etc. - // This is for the current language only. - indexPages Pages + // rawAllPages plus additional pages created during the build process. + workAllPages pageStatePages - // A convenience cache for the regular pages. - // This is for the current language only. - RegularPages Pages + // Includes headless bundles, i.e. bundles that produce no output for its content page. + headlessPages pageStatePages - // A convenience cache for the all the regular pages. - AllRegularPages Pages + // Lazy initialized page collections + pages *lazyPagesFactory + regularPages *lazyPagesFactory + allPages *lazyPagesFactory + allRegularPages *lazyPagesFactory - // Includes absolute all pages (of all types), including drafts etc. - rawAllPages Pages + // The index for .Site.GetPage etc. + pageIndex *cache.Lazy +} - // Includes headless bundles, i.e. bundles that produce no output for its content page. - headlessPages Pages +// Pages returns all pages. +// This is for the current language only. +func (c *PageCollections) Pages() page.Pages { + return c.pages.get() +} - pageIndex *cache.Lazy +// RegularPages returns all the regular pages. +// This is for the current language only. +func (c *PageCollections) RegularPages() page.Pages { + return c.regularPages.get() +} + +// AllPages returns all pages for all languages. +func (c *PageCollections) AllPages() page.Pages { + return c.allPages.get() +} + +// AllPages returns all regular pages for all languages. +func (c *PageCollections) AllRegularPages() page.Pages { + return c.allRegularPages.get() } // Get initializes the index if not already done so, then // looks up the given page ref, returns nil if no value found. -func (c *PageCollections) getFromCache(ref string) (*Page, error) { +func (c *PageCollections) getFromCache(ref string) (page.Page, error) { v, found, err := c.pageIndex.Get(ref) if err != nil { return nil, err @@ -63,7 +85,7 @@ func (c *PageCollections) getFromCache(ref string) (*Page, error) { return nil, nil } - p := v.(*Page) + p := v.(page.Page) if p != ambiguityFlag { return p, nil @@ -71,17 +93,49 @@ func (c *PageCollections) getFromCache(ref string) (*Page, error) { return nil, fmt.Errorf("page reference %q is ambiguous", ref) } -var ambiguityFlag = &Page{Kind: kindUnknown, title: "ambiguity flag"} +type lazyPagesFactory struct { + pages page.Pages -func (c *PageCollections) refreshPageCaches() { - c.indexPages = c.findPagesByKindNotIn(KindPage, c.Pages) - c.RegularPages = c.findPagesByKindIn(KindPage, c.Pages) - c.AllRegularPages = c.findPagesByKindIn(KindPage, c.AllPages) + init sync.Once + factory page.PagesFactory +} - indexLoader := func() (map[string]interface{}, error) { +func (l *lazyPagesFactory) get() page.Pages { + l.init.Do(func() { + l.pages = l.factory() + }) + return l.pages +} + +func newLazyPagesFactory(factory page.PagesFactory) *lazyPagesFactory { + return &lazyPagesFactory{factory: factory} +} + +func newPageCollections() *PageCollections { + return newPageCollectionsFromPages(nil) +} + +func newPageCollectionsFromPages(pages pageStatePages) *PageCollections { + + c := &PageCollections{rawAllPages: pages} + + c.pages = newLazyPagesFactory(func() page.Pages { + pages := make(page.Pages, len(c.workAllPages)) + for i, p := range c.workAllPages { + pages[i] = p + } + return pages + }) + + c.regularPages = newLazyPagesFactory(func() page.Pages { + return c.findPagesByKindInWorkPages(page.KindPage, c.workAllPages) + }) + + c.pageIndex = cache.NewLazy(func() (map[string]interface{}, error) { index := make(map[string]interface{}) - add := func(ref string, p *Page) { + add := func(ref string, p page.Page) { + ref = strings.ToLower(ref) existing := index[ref] if existing == nil { index[ref] = p @@ -90,71 +144,63 @@ func (c *PageCollections) refreshPageCaches() { } } - for _, pageCollection := range []Pages{c.RegularPages, c.headlessPages} { + for _, pageCollection := range []pageStatePages{c.workAllPages, c.headlessPages} { for _, p := range pageCollection { - sourceRef := p.absoluteSourceRef() + if p.IsPage() { + sourceRef := p.sourceRef() - if sourceRef != "" { - // index the canonical ref - // e.g. /section/article.md - add(sourceRef, p) - } + if sourceRef != "" { + // index the canonical ref + // e.g. /section/article.md + add(sourceRef, p) + } + + // Ref/Relref supports this potentially ambiguous lookup. + add(p.File().LogicalName(), p) - // Ref/Relref supports this potentially ambiguous lookup. - add(p.LogicalName(), p) + translationBaseName := p.File().TranslationBaseName() - translationBaseName := p.TranslationBaseName() + dir, _ := path.Split(sourceRef) + dir = strings.TrimSuffix(dir, "/") - dir, _ := path.Split(sourceRef) - dir = strings.TrimSuffix(dir, "/") + if translationBaseName == "index" { + add(dir, p) + add(path.Base(dir), p) + } else { + add(translationBaseName, p) + } - if translationBaseName == "index" { - add(dir, p) - add(path.Base(dir), p) + // We need a way to get to the current language version. + pathWithNoExtensions := path.Join(dir, translationBaseName) + add(pathWithNoExtensions, p) } else { - add(translationBaseName, p) + // index the canonical, unambiguous ref for any backing file + // e.g. /section/_index.md + sourceRef := p.sourceRef() + if sourceRef != "" { + add(sourceRef, p) + } + + ref := p.SectionsPath() + + // index the canonical, unambiguous virtual ref + // e.g. /section + // (this may already have been indexed above) + add("/"+ref, p) } - - // We need a way to get to the current language version. - pathWithNoExtensions := path.Join(dir, translationBaseName) - add(pathWithNoExtensions, p) - } - } - - for _, p := range c.indexPages { - // index the canonical, unambiguous ref for any backing file - // e.g. /section/_index.md - sourceRef := p.absoluteSourceRef() - if sourceRef != "" { - add(sourceRef, p) } - - ref := path.Join(p.sections...) - - // index the canonical, unambiguous virtual ref - // e.g. /section - // (this may already have been indexed above) - add("/"+ref, p) } return index, nil - } + }) - c.pageIndex = cache.NewLazy(indexLoader) -} - -func newPageCollections() *PageCollections { - return &PageCollections{} -} - -func newPageCollectionsFromPages(pages Pages) *PageCollections { - return &PageCollections{rawAllPages: pages} + return c } // This is an adapter func for the old API with Kind as first argument. // This is invoked when you do .Site.GetPage. We drop the Kind and fails // if there are more than 2 arguments, which would be ambigous. -func (c *PageCollections) getPageOldVersion(ref ...string) (*Page, error) { +func (c *PageCollections) getPageOldVersion(ref ...string) (page.Page, error) { var refs []string for _, r := range ref { // A common construct in the wild is @@ -173,10 +219,10 @@ func (c *PageCollections) getPageOldVersion(ref ...string) (*Page, error) { return nil, fmt.Errorf(`too many arguments to .Site.GetPage: %v. Use lookups on the form {{ .Site.GetPage "/posts/mypage-md" }}`, ref) } - if len(refs) == 0 || refs[0] == KindHome { + if len(refs) == 0 || refs[0] == page.KindHome { key = "/" } else if len(refs) == 1 { - if len(ref) == 2 && refs[0] == KindSection { + if len(ref) == 2 && refs[0] == page.KindSection { // This is an old style reference to the "Home Page section". // Typically fetched via {{ .Site.GetPage "section" .Section }} // See https://github.com/gohugoio/hugo/issues/4989 @@ -197,17 +243,18 @@ func (c *PageCollections) getPageOldVersion(ref ...string) (*Page, error) { } // Only used in tests. -func (c *PageCollections) getPage(typ string, sections ...string) *Page { +func (c *PageCollections) getPage(typ string, sections ...string) page.Page { refs := append([]string{typ}, path.Join(sections...)) p, _ := c.getPageOldVersion(refs...) return p } -// Ref is either unix-style paths (i.e. callers responsible for -// calling filepath.ToSlash as necessary) or shorthand refs. -func (c *PageCollections) getPageNew(context *Page, ref string) (*Page, error) { +// Case insensitive page lookup. +func (c *PageCollections) getPageNew(context page.Page, ref string) (page.Page, error) { var anError error + ref = strings.ToLower(ref) + // Absolute (content root relative) reference. if strings.HasPrefix(ref, "/") { p, err := c.getFromCache(ref) @@ -220,7 +267,7 @@ func (c *PageCollections) getPageNew(context *Page, ref string) (*Page, error) { } else if context != nil { // Try the page-relative path. - ppath := path.Join("/", strings.Join(context.sections, "/"), ref) + ppath := path.Join("/", strings.ToLower(context.SectionsPath()), ref) p, err := c.getFromCache(ppath) if err == nil && p != nil { return p, nil @@ -236,7 +283,8 @@ func (c *PageCollections) getPageNew(context *Page, ref string) (*Page, error) { if err == nil && p != nil { if context != nil { // TODO(bep) remove this case and the message below when the storm has passed - helpers.DistinctFeedbackLog.Printf(`WARNING: make non-relative ref/relref page reference(s) in page %q absolute, e.g. {{< ref "/blog/my-post.md" >}}`, context.absoluteSourceRef()) + err := wrapErr(errors.New(`make non-relative ref/relref page reference(s) in page %q absolute, e.g. {{< ref "/blog/my-post.md" >}}`), context) + helpers.DistinctWarnLog.Println(err) } return p, nil } @@ -253,49 +301,56 @@ func (c *PageCollections) getPageNew(context *Page, ref string) (*Page, error) { } if p == nil && anError != nil { - if context != nil { - return nil, fmt.Errorf("failed to resolve path from page %q: %s", context.absoluteSourceRef(), anError) - } - return nil, fmt.Errorf("failed to resolve page: %s", anError) + return nil, wrapErr(errors.Wrap(anError, "failed to resolve ref"), context) } return p, nil } -func (*PageCollections) findPagesByKindIn(kind string, inPages Pages) Pages { - var pages Pages +func (*PageCollections) findPagesByKindIn(kind string, inPages page.Pages) page.Pages { + var pages page.Pages for _, p := range inPages { - if p.Kind == kind { + if p.Kind() == kind { pages = append(pages, p) } } return pages } -func (*PageCollections) findFirstPageByKindIn(kind string, inPages Pages) *Page { - for _, p := range inPages { - if p.Kind == kind { - return p +func (c *PageCollections) findPagesByKind(kind string) page.Pages { + return c.findPagesByKindIn(kind, c.Pages()) +} + +func (c *PageCollections) findWorkPagesByKind(kind string) pageStatePages { + var pages pageStatePages + for _, p := range c.workAllPages { + if p.Kind() == kind { + pages = append(pages, p) } } - return nil + return pages } -func (*PageCollections) findPagesByKindNotIn(kind string, inPages Pages) Pages { - var pages Pages +func (*PageCollections) findPagesByKindInWorkPages(kind string, inPages pageStatePages) page.Pages { + var pages page.Pages for _, p := range inPages { - if p.Kind != kind { + if p.Kind() == kind { pages = append(pages, p) } } return pages } -func (c *PageCollections) findPagesByKind(kind string) Pages { - return c.findPagesByKindIn(kind, c.Pages) +func (c *PageCollections) findFirstWorkPageByKindIn(kind string) *pageState { + for _, p := range c.workAllPages { + if p.Kind() == kind { + return p + } + } + return nil } -func (c *PageCollections) addPage(page *Page) { +func (c *PageCollections) addPage(page *pageState) { c.rawAllPages = append(c.rawAllPages, page) } @@ -307,35 +362,31 @@ func (c *PageCollections) removePageFilename(filename string) { } -func (c *PageCollections) removePage(page *Page) { +func (c *PageCollections) removePage(page *pageState) { if i := c.rawAllPages.findPagePos(page); i >= 0 { c.clearResourceCacheForPage(c.rawAllPages[i]) c.rawAllPages = append(c.rawAllPages[:i], c.rawAllPages[i+1:]...) } - } -func (c *PageCollections) findPagesByShortcode(shortcode string) Pages { - var pages Pages - +func (c *PageCollections) findPagesByShortcode(shortcode string) page.Pages { + var pages page.Pages for _, p := range c.rawAllPages { - if p.shortcodeState != nil { - if _, ok := p.shortcodeState.nameSet[shortcode]; ok { - pages = append(pages, p) - } + if p.HasShortcode(shortcode) { + pages = append(pages, p) } } return pages } -func (c *PageCollections) replacePage(page *Page) { +func (c *PageCollections) replacePage(page *pageState) { // will find existing page that matches filepath and remove it c.removePage(page) c.addPage(page) } -func (c *PageCollections) clearResourceCacheForPage(page *Page) { - if len(page.Resources) > 0 { - page.s.ResourceSpec.DeleteCacheByPrefix(page.relTargetPathBase) +func (c *PageCollections) clearResourceCacheForPage(page *pageState) { + if len(page.resources) > 0 { + page.s.ResourceSpec.DeleteCacheByPrefix(page.targetPaths().SubResourceBaseTarget) } } diff --git a/hugolib/pagecollections_test.go b/hugolib/pagecollections_test.go index 2f8b31490..a5a347f83 100644 --- a/hugolib/pagecollections_test.go +++ b/hugolib/pagecollections_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,6 +21,8 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/deps" "github.com/stretchr/testify/require" ) @@ -98,12 +100,12 @@ func BenchmarkGetPageRegular(b *testing.B) { type testCase struct { kind string - context *Page + context page.Page path []string expectedTitle string } -func (t *testCase) check(p *Page, err error, errorMsg string, assert *require.Assertions) { +func (t *testCase) check(p page.Page, err error, errorMsg string, assert *require.Assertions) { switch t.kind { case "Ambiguous": assert.Error(err) @@ -114,8 +116,8 @@ func (t *testCase) check(p *Page, err error, errorMsg string, assert *require.As default: assert.NoError(err, errorMsg) assert.NotNil(p, errorMsg) - assert.Equal(t.kind, p.Kind, errorMsg) - assert.Equal(t.expectedTitle, p.title, errorMsg) + assert.Equal(t.kind, p.Kind(), errorMsg) + assert.Equal(t.expectedTitle, p.Title(), errorMsg) } } @@ -159,62 +161,62 @@ func TestGetPage(t *testing.T) { tests := []testCase{ // legacy content root relative paths - {KindHome, nil, []string{}, "home page"}, - {KindPage, nil, []string{"about.md"}, "about page"}, - {KindSection, nil, []string{"sect3"}, "section 3"}, - {KindPage, nil, []string{"sect3/page1.md"}, "Title3_1"}, - {KindPage, nil, []string{"sect4/page2.md"}, "Title4_2"}, - {KindSection, nil, []string{"sect3/sect7"}, "another sect7"}, - {KindPage, nil, []string{"sect3/subsect/deep.md"}, "deep page"}, - {KindPage, nil, []string{filepath.FromSlash("sect5/page3.md")}, "Title5_3"}, //test OS-specific path + {page.KindHome, nil, []string{}, "home page"}, + {page.KindPage, nil, []string{"about.md"}, "about page"}, + {page.KindSection, nil, []string{"sect3"}, "section 3"}, + {page.KindPage, nil, []string{"sect3/page1.md"}, "Title3_1"}, + {page.KindPage, nil, []string{"sect4/page2.md"}, "Title4_2"}, + {page.KindSection, nil, []string{"sect3/sect7"}, "another sect7"}, + {page.KindPage, nil, []string{"sect3/subsect/deep.md"}, "deep page"}, + {page.KindPage, nil, []string{filepath.FromSlash("sect5/page3.md")}, "Title5_3"}, //test OS-specific path // shorthand refs (potentially ambiguous) - {KindPage, nil, []string{"unique.md"}, "UniqueBase"}, + {page.KindPage, nil, []string{"unique.md"}, "UniqueBase"}, {"Ambiguous", nil, []string{"page1.md"}, ""}, // ISSUE: This is an ambiguous ref, but because we have to support the legacy // content root relative paths without a leading slash, the lookup // returns /sect7. This undermines ambiguity detection, but we have no choice. //{"Ambiguous", nil, []string{"sect7"}, ""}, - {KindSection, nil, []string{"sect7"}, "Sect7s"}, + {page.KindSection, nil, []string{"sect7"}, "Sect7s"}, // absolute paths - {KindHome, nil, []string{"/"}, "home page"}, - {KindPage, nil, []string{"/about.md"}, "about page"}, - {KindSection, nil, []string{"/sect3"}, "section 3"}, - {KindPage, nil, []string{"/sect3/page1.md"}, "Title3_1"}, - {KindPage, nil, []string{"/sect4/page2.md"}, "Title4_2"}, - {KindSection, nil, []string{"/sect3/sect7"}, "another sect7"}, - {KindPage, nil, []string{"/sect3/subsect/deep.md"}, "deep page"}, - {KindPage, nil, []string{filepath.FromSlash("/sect5/page3.md")}, "Title5_3"}, //test OS-specific path - {KindPage, nil, []string{"/sect3/unique.md"}, "UniqueBase"}, //next test depends on this page existing + {page.KindHome, nil, []string{"/"}, "home page"}, + {page.KindPage, nil, []string{"/about.md"}, "about page"}, + {page.KindSection, nil, []string{"/sect3"}, "section 3"}, + {page.KindPage, nil, []string{"/sect3/page1.md"}, "Title3_1"}, + {page.KindPage, nil, []string{"/sect4/page2.md"}, "Title4_2"}, + {page.KindSection, nil, []string{"/sect3/sect7"}, "another sect7"}, + {page.KindPage, nil, []string{"/sect3/subsect/deep.md"}, "deep page"}, + {page.KindPage, nil, []string{filepath.FromSlash("/sect5/page3.md")}, "Title5_3"}, //test OS-specific path + {page.KindPage, nil, []string{"/sect3/unique.md"}, "UniqueBase"}, //next test depends on this page existing // {"NoPage", nil, []string{"/unique.md"}, ""}, // ISSUE #4969: this is resolving to /sect3/unique.md {"NoPage", nil, []string{"/missing-page.md"}, ""}, {"NoPage", nil, []string{"/missing-section"}, ""}, // relative paths - {KindHome, sec3, []string{".."}, "home page"}, - {KindHome, sec3, []string{"../"}, "home page"}, - {KindPage, sec3, []string{"../about.md"}, "about page"}, - {KindSection, sec3, []string{"."}, "section 3"}, - {KindSection, sec3, []string{"./"}, "section 3"}, - {KindPage, sec3, []string{"page1.md"}, "Title3_1"}, - {KindPage, sec3, []string{"./page1.md"}, "Title3_1"}, - {KindPage, sec3, []string{"../sect4/page2.md"}, "Title4_2"}, - {KindSection, sec3, []string{"sect7"}, "another sect7"}, - {KindSection, sec3, []string{"./sect7"}, "another sect7"}, - {KindPage, sec3, []string{"./subsect/deep.md"}, "deep page"}, - {KindPage, sec3, []string{"./subsect/../../sect7/page9.md"}, "Title7_9"}, - {KindPage, sec3, []string{filepath.FromSlash("../sect5/page3.md")}, "Title5_3"}, //test OS-specific path - {KindPage, sec3, []string{"./unique.md"}, "UniqueBase"}, + {page.KindHome, sec3, []string{".."}, "home page"}, + {page.KindHome, sec3, []string{"../"}, "home page"}, + {page.KindPage, sec3, []string{"../about.md"}, "about page"}, + {page.KindSection, sec3, []string{"."}, "section 3"}, + {page.KindSection, sec3, []string{"./"}, "section 3"}, + {page.KindPage, sec3, []string{"page1.md"}, "Title3_1"}, + {page.KindPage, sec3, []string{"./page1.md"}, "Title3_1"}, + {page.KindPage, sec3, []string{"../sect4/page2.md"}, "Title4_2"}, + {page.KindSection, sec3, []string{"sect7"}, "another sect7"}, + {page.KindSection, sec3, []string{"./sect7"}, "another sect7"}, + {page.KindPage, sec3, []string{"./subsect/deep.md"}, "deep page"}, + {page.KindPage, sec3, []string{"./subsect/../../sect7/page9.md"}, "Title7_9"}, + {page.KindPage, sec3, []string{filepath.FromSlash("../sect5/page3.md")}, "Title5_3"}, //test OS-specific path + {page.KindPage, sec3, []string{"./unique.md"}, "UniqueBase"}, {"NoPage", sec3, []string{"./sect2"}, ""}, //{"NoPage", sec3, []string{"sect2"}, ""}, // ISSUE: /sect3 page relative query is resolving to /sect2 // absolute paths ignore context - {KindHome, sec3, []string{"/"}, "home page"}, - {KindPage, sec3, []string{"/about.md"}, "about page"}, - {KindPage, sec3, []string{"/sect4/page2.md"}, "Title4_2"}, - {KindPage, sec3, []string{"/sect3/subsect/deep.md"}, "deep page"}, //next test depends on this page existing + {page.KindHome, sec3, []string{"/"}, "home page"}, + {page.KindPage, sec3, []string{"/about.md"}, "about page"}, + {page.KindPage, sec3, []string{"/sect4/page2.md"}, "Title4_2"}, + {page.KindPage, sec3, []string{"/sect3/subsect/deep.md"}, "deep page"}, //next test depends on this page existing {"NoPage", sec3, []string{"/subsect/deep.md"}, ""}, } diff --git a/hugolib/pagemeta/page_frontmatter.go b/hugolib/pagemeta/page_frontmatter.go deleted file mode 100644 index b67ffbc05..000000000 --- a/hugolib/pagemeta/page_frontmatter.go +++ /dev/null @@ -1,426 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pagemeta - -import ( - "strings" - "time" - - "github.com/gohugoio/hugo/common/loggers" - "github.com/gohugoio/hugo/helpers" - - "github.com/gohugoio/hugo/config" - "github.com/spf13/cast" -) - -// FrontMatterHandler maps front matter into Page fields and .Params. -// Note that we currently have only extracted the date logic. -type FrontMatterHandler struct { - fmConfig frontmatterConfig - - dateHandler frontMatterFieldHandler - lastModHandler frontMatterFieldHandler - publishDateHandler frontMatterFieldHandler - expiryDateHandler frontMatterFieldHandler - - // A map of all date keys configured, including any custom. - allDateKeys map[string]bool - - logger *loggers.Logger -} - -// FrontMatterDescriptor describes how to handle front matter for a given Page. -// It has pointers to values in the receiving page which gets updated. -type FrontMatterDescriptor struct { - - // This the Page's front matter. - Frontmatter map[string]interface{} - - // This is the Page's base filename (BaseFilename), e.g. page.md., or - // if page is a leaf bundle, the bundle folder name (ContentBaseName). - BaseFilename string - - // The content file's mod time. - ModTime time.Time - - // May be set from the author date in Git. - GitAuthorDate time.Time - - // The below are pointers to values on Page and will be modified. - - // This is the Page's params. - Params map[string]interface{} - - // This is the Page's dates. - Dates *PageDates - - // This is the Page's Slug etc. - PageURLs *URLPath -} - -var ( - dateFieldAliases = map[string][]string{ - fmDate: {}, - fmLastmod: {"modified"}, - fmPubDate: {"pubdate", "published"}, - fmExpiryDate: {"unpublishdate"}, - } -) - -// HandleDates updates all the dates given the current configuration and the -// supplied front matter params. Note that this requires all lower-case keys -// in the params map. -func (f FrontMatterHandler) HandleDates(d *FrontMatterDescriptor) error { - if d.Dates == nil { - panic("missing dates") - } - - if f.dateHandler == nil { - panic("missing date handler") - } - - if _, err := f.dateHandler(d); err != nil { - return err - } - - if _, err := f.lastModHandler(d); err != nil { - return err - } - - if _, err := f.publishDateHandler(d); err != nil { - return err - } - - if _, err := f.expiryDateHandler(d); err != nil { - return err - } - - return nil -} - -// IsDateKey returns whether the given front matter key is considered a date by the current -// configuration. -func (f FrontMatterHandler) IsDateKey(key string) bool { - return f.allDateKeys[key] -} - -// A Zero date is a signal that the name can not be parsed. -// This follows the format as outlined in Jekyll, https://jekyllrb.com/docs/posts/: -// "Where YEAR is a four-digit number, MONTH and DAY are both two-digit numbers" -func dateAndSlugFromBaseFilename(name string) (time.Time, string) { - withoutExt, _ := helpers.FileAndExt(name) - - if len(withoutExt) < 10 { - // This can not be a date. - return time.Time{}, "" - } - - // Note: Hugo currently have no custom timezone support. - // We will have to revisit this when that is in place. - d, err := time.Parse("2006-01-02", withoutExt[:10]) - if err != nil { - return time.Time{}, "" - } - - // Be a little lenient with the format here. - slug := strings.Trim(withoutExt[10:], " -_") - - return d, slug -} - -type frontMatterFieldHandler func(d *FrontMatterDescriptor) (bool, error) - -func (f FrontMatterHandler) newChainedFrontMatterFieldHandler(handlers ...frontMatterFieldHandler) frontMatterFieldHandler { - return func(d *FrontMatterDescriptor) (bool, error) { - for _, h := range handlers { - // First successful handler wins. - success, err := h(d) - if err != nil { - f.logger.ERROR.Println(err) - } else if success { - return true, nil - } - } - return false, nil - } -} - -type frontmatterConfig struct { - date []string - lastmod []string - publishDate []string - expiryDate []string -} - -const ( - // These are all the date handler identifiers - // All identifiers not starting with a ":" maps to a front matter parameter. - fmDate = "date" - fmPubDate = "publishdate" - fmLastmod = "lastmod" - fmExpiryDate = "expirydate" - - // Gets date from filename, e.g 218-02-22-mypage.md - fmFilename = ":filename" - - // Gets date from file OS mod time. - fmModTime = ":filemodtime" - - // Gets date from Git - fmGitAuthorDate = ":git" -) - -// This is the config you get when doing nothing. -func newDefaultFrontmatterConfig() frontmatterConfig { - return frontmatterConfig{ - date: []string{fmDate, fmPubDate, fmLastmod}, - lastmod: []string{fmGitAuthorDate, fmLastmod, fmDate, fmPubDate}, - publishDate: []string{fmPubDate, fmDate}, - expiryDate: []string{fmExpiryDate}, - } -} - -func newFrontmatterConfig(cfg config.Provider) (frontmatterConfig, error) { - c := newDefaultFrontmatterConfig() - defaultConfig := c - - if cfg.IsSet("frontmatter") { - fm := cfg.GetStringMap("frontmatter") - for k, v := range fm { - loki := strings.ToLower(k) - switch loki { - case fmDate: - c.date = toLowerSlice(v) - case fmPubDate: - c.publishDate = toLowerSlice(v) - case fmLastmod: - c.lastmod = toLowerSlice(v) - case fmExpiryDate: - c.expiryDate = toLowerSlice(v) - } - } - } - - expander := func(c, d []string) []string { - out := expandDefaultValues(c, d) - out = addDateFieldAliases(out) - return out - } - - c.date = expander(c.date, defaultConfig.date) - c.publishDate = expander(c.publishDate, defaultConfig.publishDate) - c.lastmod = expander(c.lastmod, defaultConfig.lastmod) - c.expiryDate = expander(c.expiryDate, defaultConfig.expiryDate) - - return c, nil -} - -func addDateFieldAliases(values []string) []string { - var complete []string - - for _, v := range values { - complete = append(complete, v) - if aliases, found := dateFieldAliases[v]; found { - complete = append(complete, aliases...) - } - } - return helpers.UniqueStrings(complete) -} - -func expandDefaultValues(values []string, defaults []string) []string { - var out []string - for _, v := range values { - if v == ":default" { - out = append(out, defaults...) - } else { - out = append(out, v) - } - } - return out -} - -func toLowerSlice(in interface{}) []string { - out := cast.ToStringSlice(in) - for i := 0; i < len(out); i++ { - out[i] = strings.ToLower(out[i]) - } - - return out -} - -// NewFrontmatterHandler creates a new FrontMatterHandler with the given logger and configuration. -// If no logger is provided, one will be created. -func NewFrontmatterHandler(logger *loggers.Logger, cfg config.Provider) (FrontMatterHandler, error) { - - if logger == nil { - logger = loggers.NewWarningLogger() - } - - frontMatterConfig, err := newFrontmatterConfig(cfg) - if err != nil { - return FrontMatterHandler{}, err - } - - allDateKeys := make(map[string]bool) - addKeys := func(vals []string) { - for _, k := range vals { - if !strings.HasPrefix(k, ":") { - allDateKeys[k] = true - } - } - } - - addKeys(frontMatterConfig.date) - addKeys(frontMatterConfig.expiryDate) - addKeys(frontMatterConfig.lastmod) - addKeys(frontMatterConfig.publishDate) - - f := FrontMatterHandler{logger: logger, fmConfig: frontMatterConfig, allDateKeys: allDateKeys} - - if err := f.createHandlers(); err != nil { - return f, err - } - - return f, nil -} - -func (f *FrontMatterHandler) createHandlers() error { - var err error - - if f.dateHandler, err = f.createDateHandler(f.fmConfig.date, - func(d *FrontMatterDescriptor, t time.Time) { - d.Dates.Date = t - setParamIfNotSet(fmDate, t, d) - }); err != nil { - return err - } - - if f.lastModHandler, err = f.createDateHandler(f.fmConfig.lastmod, - func(d *FrontMatterDescriptor, t time.Time) { - setParamIfNotSet(fmLastmod, t, d) - d.Dates.Lastmod = t - }); err != nil { - return err - } - - if f.publishDateHandler, err = f.createDateHandler(f.fmConfig.publishDate, - func(d *FrontMatterDescriptor, t time.Time) { - setParamIfNotSet(fmPubDate, t, d) - d.Dates.PublishDate = t - }); err != nil { - return err - } - - if f.expiryDateHandler, err = f.createDateHandler(f.fmConfig.expiryDate, - func(d *FrontMatterDescriptor, t time.Time) { - setParamIfNotSet(fmExpiryDate, t, d) - d.Dates.ExpiryDate = t - }); err != nil { - return err - } - - return nil -} - -func setParamIfNotSet(key string, value interface{}, d *FrontMatterDescriptor) { - if _, found := d.Params[key]; found { - return - } - d.Params[key] = value -} - -func (f FrontMatterHandler) createDateHandler(identifiers []string, setter func(d *FrontMatterDescriptor, t time.Time)) (frontMatterFieldHandler, error) { - var h *frontmatterFieldHandlers - var handlers []frontMatterFieldHandler - - for _, identifier := range identifiers { - switch identifier { - case fmFilename: - handlers = append(handlers, h.newDateFilenameHandler(setter)) - case fmModTime: - handlers = append(handlers, h.newDateModTimeHandler(setter)) - case fmGitAuthorDate: - handlers = append(handlers, h.newDateGitAuthorDateHandler(setter)) - default: - handlers = append(handlers, h.newDateFieldHandler(identifier, setter)) - } - } - - return f.newChainedFrontMatterFieldHandler(handlers...), nil - -} - -type frontmatterFieldHandlers int - -func (f *frontmatterFieldHandlers) newDateFieldHandler(key string, setter func(d *FrontMatterDescriptor, t time.Time)) frontMatterFieldHandler { - return func(d *FrontMatterDescriptor) (bool, error) { - v, found := d.Frontmatter[key] - - if !found { - return false, nil - } - - date, err := cast.ToTimeE(v) - if err != nil { - return false, nil - } - - // We map several date keys to one, so, for example, - // "expirydate", "unpublishdate" will all set .ExpiryDate (first found). - setter(d, date) - - // This is the params key as set in front matter. - d.Params[key] = date - - return true, nil - } -} - -func (f *frontmatterFieldHandlers) newDateFilenameHandler(setter func(d *FrontMatterDescriptor, t time.Time)) frontMatterFieldHandler { - return func(d *FrontMatterDescriptor) (bool, error) { - date, slug := dateAndSlugFromBaseFilename(d.BaseFilename) - if date.IsZero() { - return false, nil - } - - setter(d, date) - - if _, found := d.Frontmatter["slug"]; !found { - // Use slug from filename - d.PageURLs.Slug = slug - } - - return true, nil - } -} - -func (f *frontmatterFieldHandlers) newDateModTimeHandler(setter func(d *FrontMatterDescriptor, t time.Time)) frontMatterFieldHandler { - return func(d *FrontMatterDescriptor) (bool, error) { - if d.ModTime.IsZero() { - return false, nil - } - setter(d, d.ModTime) - return true, nil - } -} - -func (f *frontmatterFieldHandlers) newDateGitAuthorDateHandler(setter func(d *FrontMatterDescriptor, t time.Time)) frontMatterFieldHandler { - return func(d *FrontMatterDescriptor) (bool, error) { - if d.GitAuthorDate.IsZero() { - return false, nil - } - setter(d, d.GitAuthorDate) - return true, nil - } -} diff --git a/hugolib/pagemeta/page_frontmatter_test.go b/hugolib/pagemeta/page_frontmatter_test.go deleted file mode 100644 index 03f4c2f84..000000000 --- a/hugolib/pagemeta/page_frontmatter_test.go +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pagemeta - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/spf13/viper" - - "github.com/stretchr/testify/require" -) - -func TestDateAndSlugFromBaseFilename(t *testing.T) { - - t.Parallel() - - assert := require.New(t) - - tests := []struct { - name string - date string - slug string - }{ - {"page.md", "0001-01-01", ""}, - {"2012-09-12-page.md", "2012-09-12", "page"}, - {"2018-02-28-page.md", "2018-02-28", "page"}, - {"2018-02-28_page.md", "2018-02-28", "page"}, - {"2018-02-28 page.md", "2018-02-28", "page"}, - {"2018-02-28page.md", "2018-02-28", "page"}, - {"2018-02-28-.md", "2018-02-28", ""}, - {"2018-02-28-.md", "2018-02-28", ""}, - {"2018-02-28.md", "2018-02-28", ""}, - {"2018-02-28-page", "2018-02-28", "page"}, - {"2012-9-12-page.md", "0001-01-01", ""}, - {"asdfasdf.md", "0001-01-01", ""}, - } - - for i, test := range tests { - expectedDate, err := time.Parse("2006-01-02", test.date) - assert.NoError(err) - - errMsg := fmt.Sprintf("Test %d", i) - gotDate, gotSlug := dateAndSlugFromBaseFilename(test.name) - - assert.Equal(expectedDate, gotDate, errMsg) - assert.Equal(test.slug, gotSlug, errMsg) - - } -} - -func newTestFd() *FrontMatterDescriptor { - return &FrontMatterDescriptor{ - Frontmatter: make(map[string]interface{}), - Params: make(map[string]interface{}), - Dates: &PageDates{}, - PageURLs: &URLPath{}, - } -} - -func TestFrontMatterNewConfig(t *testing.T) { - assert := require.New(t) - - cfg := viper.New() - - cfg.Set("frontmatter", map[string]interface{}{ - "date": []string{"publishDate", "LastMod"}, - "Lastmod": []string{"publishDate"}, - "expiryDate": []string{"lastMod"}, - "publishDate": []string{"date"}, - }) - - fc, err := newFrontmatterConfig(cfg) - assert.NoError(err) - assert.Equal([]string{"publishdate", "pubdate", "published", "lastmod", "modified"}, fc.date) - assert.Equal([]string{"publishdate", "pubdate", "published"}, fc.lastmod) - assert.Equal([]string{"lastmod", "modified"}, fc.expiryDate) - assert.Equal([]string{"date"}, fc.publishDate) - - // Default - cfg = viper.New() - fc, err = newFrontmatterConfig(cfg) - assert.NoError(err) - assert.Equal([]string{"date", "publishdate", "pubdate", "published", "lastmod", "modified"}, fc.date) - assert.Equal([]string{":git", "lastmod", "modified", "date", "publishdate", "pubdate", "published"}, fc.lastmod) - assert.Equal([]string{"expirydate", "unpublishdate"}, fc.expiryDate) - assert.Equal([]string{"publishdate", "pubdate", "published", "date"}, fc.publishDate) - - // :default keyword - cfg.Set("frontmatter", map[string]interface{}{ - "date": []string{"d1", ":default"}, - "lastmod": []string{"d2", ":default"}, - "expiryDate": []string{"d3", ":default"}, - "publishDate": []string{"d4", ":default"}, - }) - fc, err = newFrontmatterConfig(cfg) - assert.NoError(err) - assert.Equal([]string{"d1", "date", "publishdate", "pubdate", "published", "lastmod", "modified"}, fc.date) - assert.Equal([]string{"d2", ":git", "lastmod", "modified", "date", "publishdate", "pubdate", "published"}, fc.lastmod) - assert.Equal([]string{"d3", "expirydate", "unpublishdate"}, fc.expiryDate) - assert.Equal([]string{"d4", "publishdate", "pubdate", "published", "date"}, fc.publishDate) - -} - -func TestFrontMatterDatesHandlers(t *testing.T) { - assert := require.New(t) - - for _, handlerID := range []string{":filename", ":fileModTime", ":git"} { - - cfg := viper.New() - - cfg.Set("frontmatter", map[string]interface{}{ - "date": []string{handlerID, "date"}, - }) - - handler, err := NewFrontmatterHandler(nil, cfg) - assert.NoError(err) - - d1, _ := time.Parse("2006-01-02", "2018-02-01") - d2, _ := time.Parse("2006-01-02", "2018-02-02") - - d := newTestFd() - switch strings.ToLower(handlerID) { - case ":filename": - d.BaseFilename = "2018-02-01-page.md" - case ":filemodtime": - d.ModTime = d1 - case ":git": - d.GitAuthorDate = d1 - } - d.Frontmatter["date"] = d2 - assert.NoError(handler.HandleDates(d)) - assert.Equal(d1, d.Dates.Date) - assert.Equal(d2, d.Params["date"]) - - d = newTestFd() - d.Frontmatter["date"] = d2 - assert.NoError(handler.HandleDates(d)) - assert.Equal(d2, d.Dates.Date) - assert.Equal(d2, d.Params["date"]) - - } -} - -func TestFrontMatterDatesCustomConfig(t *testing.T) { - t.Parallel() - - assert := require.New(t) - - cfg := viper.New() - cfg.Set("frontmatter", map[string]interface{}{ - "date": []string{"mydate"}, - "lastmod": []string{"publishdate"}, - "publishdate": []string{"publishdate"}, - }) - - handler, err := NewFrontmatterHandler(nil, cfg) - assert.NoError(err) - - testDate, err := time.Parse("2006-01-02", "2018-02-01") - assert.NoError(err) - - d := newTestFd() - d.Frontmatter["mydate"] = testDate - testDate = testDate.Add(24 * time.Hour) - d.Frontmatter["date"] = testDate - testDate = testDate.Add(24 * time.Hour) - d.Frontmatter["lastmod"] = testDate - testDate = testDate.Add(24 * time.Hour) - d.Frontmatter["publishdate"] = testDate - testDate = testDate.Add(24 * time.Hour) - d.Frontmatter["expirydate"] = testDate - - assert.NoError(handler.HandleDates(d)) - - assert.Equal(1, d.Dates.Date.Day()) - assert.Equal(4, d.Dates.Lastmod.Day()) - assert.Equal(4, d.Dates.PublishDate.Day()) - assert.Equal(5, d.Dates.ExpiryDate.Day()) - - assert.Equal(d.Dates.Date, d.Params["date"]) - assert.Equal(d.Dates.Date, d.Params["mydate"]) - assert.Equal(d.Dates.PublishDate, d.Params["publishdate"]) - assert.Equal(d.Dates.ExpiryDate, d.Params["expirydate"]) - - assert.False(handler.IsDateKey("date")) // This looks odd, but is configured like this. - assert.True(handler.IsDateKey("mydate")) - assert.True(handler.IsDateKey("publishdate")) - assert.True(handler.IsDateKey("pubdate")) - -} - -func TestFrontMatterDatesDefaultKeyword(t *testing.T) { - t.Parallel() - - assert := require.New(t) - - cfg := viper.New() - - cfg.Set("frontmatter", map[string]interface{}{ - "date": []string{"mydate", ":default"}, - "publishdate": []string{":default", "mypubdate"}, - }) - - handler, err := NewFrontmatterHandler(nil, cfg) - assert.NoError(err) - - testDate, _ := time.Parse("2006-01-02", "2018-02-01") - d := newTestFd() - d.Frontmatter["mydate"] = testDate - d.Frontmatter["date"] = testDate.Add(1 * 24 * time.Hour) - d.Frontmatter["mypubdate"] = testDate.Add(2 * 24 * time.Hour) - d.Frontmatter["publishdate"] = testDate.Add(3 * 24 * time.Hour) - - assert.NoError(handler.HandleDates(d)) - - assert.Equal(1, d.Dates.Date.Day()) - assert.Equal(2, d.Dates.Lastmod.Day()) - assert.Equal(4, d.Dates.PublishDate.Day()) - assert.True(d.Dates.ExpiryDate.IsZero()) - -} - -func TestExpandDefaultValues(t *testing.T) { - assert := require.New(t) - assert.Equal([]string{"a", "b", "c", "d"}, expandDefaultValues([]string{"a", ":default", "d"}, []string{"b", "c"})) - assert.Equal([]string{"a", "b", "c"}, expandDefaultValues([]string{"a", "b", "c"}, []string{"a", "b", "c"})) - assert.Equal([]string{"b", "c", "a", "b", "c", "d"}, expandDefaultValues([]string{":default", "a", ":default", "d"}, []string{"b", "c"})) - -} - -func TestFrontMatterDateFieldHandler(t *testing.T) { - t.Parallel() - - assert := require.New(t) - - handlers := new(frontmatterFieldHandlers) - - fd := newTestFd() - d, _ := time.Parse("2006-01-02", "2018-02-01") - fd.Frontmatter["date"] = d - h := handlers.newDateFieldHandler("date", func(d *FrontMatterDescriptor, t time.Time) { d.Dates.Date = t }) - - handled, err := h(fd) - assert.True(handled) - assert.NoError(err) - assert.Equal(d, fd.Dates.Date) -} diff --git a/hugolib/pagemeta/pagemeta.go b/hugolib/pagemeta/pagemeta.go deleted file mode 100644 index 93dc9a12f..000000000 --- a/hugolib/pagemeta/pagemeta.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pagemeta - -import ( - "time" -) - -type URLPath struct { - URL string - Permalink string - Slug string - Section string -} - -type PageDates struct { - Date time.Time - Lastmod time.Time - PublishDate time.Time - ExpiryDate time.Time -} diff --git a/hugolib/pagesPrevNext.go b/hugolib/pagesPrevNext.go deleted file mode 100644 index 947a49b85..000000000 --- a/hugolib/pagesPrevNext.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -// Prev returns the previous page reletive to the given page. -func (p Pages) Prev(cur *Page) *Page { - for x, c := range p { - if c.Eq(cur) { - if x == 0 { - // TODO(bep) consider return nil here to get it line with the other Prevs - return p[len(p)-1] - } - return p[x-1] - } - } - return nil -} - -// Next returns the next page reletive to the given page. -func (p Pages) Next(cur *Page) *Page { - for x, c := range p { - if c.Eq(cur) { - if x < len(p)-1 { - return p[x+1] - } - // TODO(bep) consider return nil here to get it line with the other Nexts - return p[0] - } - } - return nil -} diff --git a/hugolib/pagesPrevNext_test.go b/hugolib/pagesPrevNext_test.go deleted file mode 100644 index 5945d8fe5..000000000 --- a/hugolib/pagesPrevNext_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "testing" - - "github.com/spf13/cast" - "github.com/stretchr/testify/assert" -) - -type pagePNTestObject struct { - path string - weight int - date string -} - -var pagePNTestSources = []pagePNTestObject{ - {"/section1/testpage1.md", 5, "2012-04-06"}, - {"/section1/testpage2.md", 4, "2012-01-01"}, - {"/section1/testpage3.md", 3, "2012-04-06"}, - {"/section2/testpage4.md", 2, "2012-03-02"}, - {"/section2/testpage5.md", 1, "2012-04-06"}, -} - -func TestPrev(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - assert.Equal(t, pages.Prev(pages[0]), pages[4]) - assert.Equal(t, pages.Prev(pages[1]), pages[0]) - assert.Equal(t, pages.Prev(pages[4]), pages[3]) -} - -func TestNext(t *testing.T) { - t.Parallel() - pages := preparePageGroupTestPages(t) - assert.Equal(t, pages.Next(pages[0]), pages[1]) - assert.Equal(t, pages.Next(pages[1]), pages[2]) - assert.Equal(t, pages.Next(pages[4]), pages[0]) -} - -func prepareWeightedPagesPrevNext(t *testing.T) WeightedPages { - s := newTestSite(t) - w := WeightedPages{} - - for _, src := range pagePNTestSources { - p, err := s.NewPage(src.path) - if err != nil { - t.Fatalf("failed to prepare test page %s", src.path) - } - p.Weight = src.weight - p.Date = cast.ToTime(src.date) - p.PublishDate = cast.ToTime(src.date) - w = append(w, WeightedPage{p.Weight, p}) - } - - w.Sort() - return w -} - -func TestWeightedPagesPrev(t *testing.T) { - t.Parallel() - w := prepareWeightedPagesPrevNext(t) - assert.Equal(t, w.Prev(w[0].Page), w[4].Page) - assert.Equal(t, w.Prev(w[1].Page), w[0].Page) - assert.Equal(t, w.Prev(w[4].Page), w[3].Page) -} - -func TestWeightedPagesNext(t *testing.T) { - t.Parallel() - w := prepareWeightedPagesPrevNext(t) - assert.Equal(t, w.Next(w[0].Page), w[1].Page) - assert.Equal(t, w.Next(w[1].Page), w[2].Page) - assert.Equal(t, w.Next(w[4].Page), w[0].Page) -} diff --git a/hugolib/pages_language_merge.go b/hugolib/pages_language_merge.go deleted file mode 100644 index 8bbae9a12..000000000 --- a/hugolib/pages_language_merge.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" -) - -var ( - _ pagesLanguageMerger = (*Pages)(nil) -) - -type pagesLanguageMerger interface { - MergeByLanguage(other Pages) Pages - // Needed for integration with the tpl package. - MergeByLanguageInterface(other interface{}) (interface{}, error) -} - -// MergeByLanguage supplies missing translations in p1 with values from p2. -// The result is sorted by the default sort order for pages. -func (p1 Pages) MergeByLanguage(p2 Pages) Pages { - merge := func(pages *Pages) { - m := make(map[string]bool) - for _, p := range *pages { - m[p.TranslationKey()] = true - } - - for _, p := range p2 { - if _, found := m[p.TranslationKey()]; !found { - *pages = append(*pages, p) - } - } - - pages.sort() - } - - out, _ := spc.getP("pages.MergeByLanguage", merge, p1, p2) - - return out -} - -// MergeByLanguageInterface is the generic version of MergeByLanguage. It -// is here just so it can be called from the tpl package. -func (p1 Pages) MergeByLanguageInterface(in interface{}) (interface{}, error) { - if in == nil { - return p1, nil - } - p2, ok := in.(Pages) - if !ok { - return nil, fmt.Errorf("%T cannot be merged by language", in) - } - return p1.MergeByLanguage(p2), nil -} diff --git a/hugolib/pages_language_merge_test.go b/hugolib/pages_language_merge_test.go index efcfbf04b..bae2ddd81 100644 --- a/hugolib/pages_language_merge_test.go +++ b/hugolib/pages_language_merge_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,6 +21,8 @@ import ( "github.com/stretchr/testify/require" ) +// TODO(bep) move and rewrite in resource/page. + func TestMergeLanguages(t *testing.T) { t.Parallel() assert := require.New(t) @@ -36,12 +38,12 @@ func TestMergeLanguages(t *testing.T) { frSite := h.Sites[1] nnSite := h.Sites[2] - assert.Equal(31, len(enSite.RegularPages)) - assert.Equal(6, len(frSite.RegularPages)) - assert.Equal(12, len(nnSite.RegularPages)) + assert.Equal(31, len(enSite.RegularPages())) + assert.Equal(6, len(frSite.RegularPages())) + assert.Equal(12, len(nnSite.RegularPages())) for i := 0; i < 2; i++ { - mergedNN := nnSite.RegularPages.MergeByLanguage(enSite.RegularPages) + mergedNN := nnSite.RegularPages().MergeByLanguage(enSite.RegularPages()) assert.Equal(31, len(mergedNN)) for i := 1; i <= 31; i++ { expectedLang := "en" @@ -49,11 +51,11 @@ func TestMergeLanguages(t *testing.T) { expectedLang = "nn" } p := mergedNN[i-1] - assert.Equal(expectedLang, p.Lang(), fmt.Sprintf("Test %d", i)) + assert.Equal(expectedLang, p.Language().Lang, fmt.Sprintf("Test %d", i)) } } - mergedFR := frSite.RegularPages.MergeByLanguage(enSite.RegularPages) + mergedFR := frSite.RegularPages().MergeByLanguage(enSite.RegularPages()) assert.Equal(31, len(mergedFR)) for i := 1; i <= 31; i++ { expectedLang := "en" @@ -61,28 +63,28 @@ func TestMergeLanguages(t *testing.T) { expectedLang = "fr" } p := mergedFR[i-1] - assert.Equal(expectedLang, p.Lang(), fmt.Sprintf("Test %d", i)) + assert.Equal(expectedLang, p.Language().Lang, fmt.Sprintf("Test %d", i)) } - firstNN := nnSite.RegularPages[0] + firstNN := nnSite.RegularPages()[0] assert.Equal(4, len(firstNN.Sites())) assert.Equal("en", firstNN.Sites().First().Language().Lang) nnBundle := nnSite.getPage("page", "bundle") enBundle := enSite.getPage("page", "bundle") - assert.Equal(6, len(enBundle.Resources)) - assert.Equal(2, len(nnBundle.Resources)) + assert.Equal(6, len(enBundle.Resources())) + assert.Equal(2, len(nnBundle.Resources())) - var ri interface{} = nnBundle.Resources + var ri interface{} = nnBundle.Resources() // This looks less ugly in the templates ... - mergedNNResources := ri.(resource.ResourcesLanguageMerger).MergeByLanguage(enBundle.Resources) + mergedNNResources := ri.(resource.ResourcesLanguageMerger).MergeByLanguage(enBundle.Resources()) assert.Equal(6, len(mergedNNResources)) - unchanged, err := nnSite.RegularPages.MergeByLanguageInterface(nil) + unchanged, err := nnSite.RegularPages().MergeByLanguageInterface(nil) assert.NoError(err) - assert.Equal(nnSite.RegularPages, unchanged) + assert.Equal(nnSite.RegularPages(), unchanged) } @@ -93,7 +95,7 @@ func TestMergeLanguagesTemplate(t *testing.T) { b.WithTemplates("home.html", ` {{ $pages := .Site.RegularPages }} {{ .Scratch.Set "pages" $pages }} -{{ if eq .Lang "nn" }}: +{{ if eq .Language.Lang "nn" }}: {{ $enSite := index .Sites 0 }} {{ $frSite := index .Sites 1 }} {{ $nnBundle := .Site.GetPage "page" "bundle" }} @@ -103,8 +105,8 @@ func TestMergeLanguagesTemplate(t *testing.T) { {{ end }} {{ $pages := .Scratch.Get "pages" }} {{ $pages2 := .Scratch.Get "pages2" }} -Pages1: {{ range $i, $p := $pages }}{{ add $i 1 }}: {{ .Path }} {{ .Lang }} | {{ end }} -Pages2: {{ range $i, $p := $pages2 }}{{ add $i 1 }}: {{ .Title }} {{ .Lang }} | {{ end }} +Pages1: {{ range $i, $p := $pages }}{{ add $i 1 }}: {{ .File.Path }} {{ .Language.Lang }} | {{ end }} +Pages2: {{ range $i, $p := $pages2 }}{{ add $i 1 }}: {{ .Title }} {{ .Language.Lang }} | {{ end }} `, "shortcodes/shortcode.html", "MyShort", @@ -178,7 +180,7 @@ func BenchmarkMergeByLanguage(b *testing.B) { nnSite := h.Sites[2] for i := 0; i < b.N; i++ { - merged := nnSite.RegularPages.MergeByLanguage(enSite.RegularPages) + merged := nnSite.RegularPages().MergeByLanguage(enSite.RegularPages()) if len(merged) != count { b.Fatal("Count mismatch") } diff --git a/hugolib/pages_related.go b/hugolib/pages_related.go deleted file mode 100644 index 2881a45e6..000000000 --- a/hugolib/pages_related.go +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "sync" - - "github.com/gohugoio/hugo/common/types" - "github.com/gohugoio/hugo/related" - "github.com/spf13/cast" -) - -var ( - // Assert that Pages and PageGroup implements the PageGenealogist interface. - _ PageGenealogist = (Pages)(nil) - _ PageGenealogist = PageGroup{} -) - -// A PageGenealogist finds related pages in a page collection. This interface is implemented -// by Pages and PageGroup, which makes it available as `{{ .RegularPages.Related . }}` etc. -type PageGenealogist interface { - - // Template example: - // {{ $related := .RegularPages.Related . }} - Related(doc related.Document) (Pages, error) - - // Template example: - // {{ $related := .RegularPages.RelatedIndices . "tags" "date" }} - RelatedIndices(doc related.Document, indices ...interface{}) (Pages, error) - - // Template example: - // {{ $related := .RegularPages.RelatedTo ( keyVals "tags" "hugo", "rocks") ( keyVals "date" .Date ) }} - RelatedTo(args ...types.KeyValues) (Pages, error) -} - -// Related searches all the configured indices with the search keywords from the -// supplied document. -func (p Pages) Related(doc related.Document) (Pages, error) { - page, err := unwrapPage(doc) - if err != nil { - return nil, err - } - - result, err := p.searchDoc(page) - if err != nil { - return nil, err - } - - return result.removeFirstIfFound(page), nil -} - -// RelatedIndices searches the given indices with the search keywords from the -// supplied document. -func (p Pages) RelatedIndices(doc related.Document, indices ...interface{}) (Pages, error) { - page, err := unwrapPage(doc) - if err != nil { - return nil, err - } - - indicesStr, err := cast.ToStringSliceE(indices) - if err != nil { - return nil, err - } - - result, err := p.searchDoc(doc, indicesStr...) - if err != nil { - return nil, err - } - - return result.removeFirstIfFound(page), nil - -} - -// RelatedTo searches the given indices with the corresponding values. -func (p Pages) RelatedTo(args ...types.KeyValues) (Pages, error) { - if len(p) == 0 { - return nil, nil - } - - return p.search(args...) - -} - -func (p Pages) search(args ...types.KeyValues) (Pages, error) { - return p.withInvertedIndex(func(idx *related.InvertedIndex) ([]related.Document, error) { - return idx.SearchKeyValues(args...) - }) - -} - -func (p Pages) searchDoc(doc related.Document, indices ...string) (Pages, error) { - return p.withInvertedIndex(func(idx *related.InvertedIndex) ([]related.Document, error) { - return idx.SearchDoc(doc, indices...) - }) -} - -func (p Pages) withInvertedIndex(search func(idx *related.InvertedIndex) ([]related.Document, error)) (Pages, error) { - if len(p) == 0 { - return nil, nil - } - - cache := p[0].s.relatedDocsHandler - - searchIndex, err := cache.getOrCreateIndex(p) - if err != nil { - return nil, err - } - - result, err := search(searchIndex) - if err != nil { - return nil, err - } - - if len(result) > 0 { - mp := make(Pages, len(result)) - for i, match := range result { - mp[i] = match.(*Page) - } - return mp, nil - } - - return nil, nil -} - -type cachedPostingList struct { - p Pages - - postingList *related.InvertedIndex -} - -type relatedDocsHandler struct { - // This is configured in site or langugage config. - cfg related.Config - - postingLists []*cachedPostingList - mu sync.RWMutex -} - -func newSearchIndexHandler(cfg related.Config) *relatedDocsHandler { - return &relatedDocsHandler{cfg: cfg} -} - -// This assumes that a lock has been acquired. -func (s *relatedDocsHandler) getIndex(p Pages) *related.InvertedIndex { - for _, ci := range s.postingLists { - if pagesEqual(p, ci.p) { - return ci.postingList - } - } - return nil -} - -func (s *relatedDocsHandler) getOrCreateIndex(p Pages) (*related.InvertedIndex, error) { - s.mu.RLock() - cachedIndex := s.getIndex(p) - if cachedIndex != nil { - s.mu.RUnlock() - return cachedIndex, nil - } - s.mu.RUnlock() - - s.mu.Lock() - defer s.mu.Unlock() - - if cachedIndex := s.getIndex(p); cachedIndex != nil { - return cachedIndex, nil - } - - searchIndex := related.NewInvertedIndex(s.cfg) - - for _, page := range p { - if err := searchIndex.Add(page); err != nil { - return nil, err - } - } - - s.postingLists = append(s.postingLists, &cachedPostingList{p: p, postingList: searchIndex}) - - return searchIndex, nil -} diff --git a/hugolib/pages_related_test.go b/hugolib/pages_related_test.go deleted file mode 100644 index ed8d9df9d..000000000 --- a/hugolib/pages_related_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "path/filepath" - "testing" - - "github.com/gohugoio/hugo/common/types" - "github.com/gohugoio/hugo/deps" - - "github.com/stretchr/testify/require" -) - -func TestRelated(t *testing.T) { - assert := require.New(t) - - t.Parallel() - - var ( - cfg, fs = newTestCfg() - //th = testHelper{cfg, fs, t} - ) - - pageTmpl := `--- -title: Page %d -keywords: [%s] -date: %s ---- - -Content -` - - writeSource(t, fs, filepath.Join("content", "page1.md"), fmt.Sprintf(pageTmpl, 1, "hugo, says", "2017-01-03")) - writeSource(t, fs, filepath.Join("content", "page2.md"), fmt.Sprintf(pageTmpl, 2, "hugo, rocks", "2017-01-02")) - writeSource(t, fs, filepath.Join("content", "page3.md"), fmt.Sprintf(pageTmpl, 3, "bep, says", "2017-01-01")) - - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - assert.Len(s.RegularPages, 3) - - result, err := s.RegularPages.RelatedTo(types.NewKeyValuesStrings("keywords", "hugo", "rocks")) - - assert.NoError(err) - assert.Len(result, 2) - assert.Equal("Page 2", result[0].title) - assert.Equal("Page 1", result[1].title) - - result, err = s.RegularPages.Related(s.RegularPages[0]) - assert.Len(result, 2) - assert.Equal("Page 2", result[0].title) - assert.Equal("Page 3", result[1].title) - - result, err = s.RegularPages.RelatedIndices(s.RegularPages[0], "keywords") - assert.Len(result, 2) - assert.Equal("Page 2", result[0].title) - assert.Equal("Page 3", result[1].title) - - result, err = s.RegularPages.RelatedTo(types.NewKeyValuesStrings("keywords", "bep", "rocks")) - assert.NoError(err) - assert.Len(result, 2) - assert.Equal("Page 2", result[0].title) - assert.Equal("Page 3", result[1].title) -} diff --git a/hugolib/pagination.go b/hugolib/pagination.go deleted file mode 100644 index 05846a6bb..000000000 --- a/hugolib/pagination.go +++ /dev/null @@ -1,595 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "errors" - "fmt" - "html/template" - "math" - "reflect" - "strings" - - "github.com/gohugoio/hugo/config" - - "github.com/spf13/cast" -) - -// Pager represents one of the elements in a paginator. -// The number, starting on 1, represents its place. -type Pager struct { - number int - *paginator -} - -func (p Pager) String() string { - return fmt.Sprintf("Pager %d", p.number) -} - -type paginatedElement interface { - Len() int -} - -// Len returns the number of pages in the list. -func (p Pages) Len() int { - return len(p) -} - -// Len returns the number of pages in the page group. -func (psg PagesGroup) Len() int { - l := 0 - for _, pg := range psg { - l += len(pg.Pages) - } - return l -} - -type pagers []*Pager - -var ( - paginatorEmptyPages Pages - paginatorEmptyPageGroups PagesGroup -) - -type paginator struct { - paginatedElements []paginatedElement - pagers - paginationURLFactory - total int - size int - source interface{} - options []interface{} -} - -type paginationURLFactory func(int) string - -// PageNumber returns the current page's number in the pager sequence. -func (p *Pager) PageNumber() int { - return p.number -} - -// URL returns the URL to the current page. -func (p *Pager) URL() template.HTML { - return template.HTML(p.paginationURLFactory(p.PageNumber())) -} - -// Pages returns the Pages on this page. -// Note: If this return a non-empty result, then PageGroups() will return empty. -func (p *Pager) Pages() Pages { - if len(p.paginatedElements) == 0 { - return paginatorEmptyPages - } - - if pages, ok := p.element().(Pages); ok { - return pages - } - - return paginatorEmptyPages -} - -// PageGroups return Page groups for this page. -// Note: If this return non-empty result, then Pages() will return empty. -func (p *Pager) PageGroups() PagesGroup { - if len(p.paginatedElements) == 0 { - return paginatorEmptyPageGroups - } - - if groups, ok := p.element().(PagesGroup); ok { - return groups - } - - return paginatorEmptyPageGroups -} - -func (p *Pager) element() paginatedElement { - if len(p.paginatedElements) == 0 { - return paginatorEmptyPages - } - return p.paginatedElements[p.PageNumber()-1] -} - -// page returns the Page with the given index -func (p *Pager) page(index int) (*Page, error) { - - if pages, ok := p.element().(Pages); ok { - if pages != nil && len(pages) > index { - return pages[index], nil - } - return nil, nil - } - - // must be PagesGroup - // this construction looks clumsy, but ... - // ... it is the difference between 99.5% and 100% test coverage :-) - groups := p.element().(PagesGroup) - - i := 0 - for _, v := range groups { - for _, page := range v.Pages { - if i == index { - return page, nil - } - i++ - } - } - return nil, nil -} - -// NumberOfElements gets the number of elements on this page. -func (p *Pager) NumberOfElements() int { - return p.element().Len() -} - -// HasPrev tests whether there are page(s) before the current. -func (p *Pager) HasPrev() bool { - return p.PageNumber() > 1 -} - -// Prev returns the pager for the previous page. -func (p *Pager) Prev() *Pager { - if !p.HasPrev() { - return nil - } - return p.pagers[p.PageNumber()-2] -} - -// HasNext tests whether there are page(s) after the current. -func (p *Pager) HasNext() bool { - return p.PageNumber() < len(p.paginatedElements) -} - -// Next returns the pager for the next page. -func (p *Pager) Next() *Pager { - if !p.HasNext() { - return nil - } - return p.pagers[p.PageNumber()] -} - -// First returns the pager for the first page. -func (p *Pager) First() *Pager { - return p.pagers[0] -} - -// Last returns the pager for the last page. -func (p *Pager) Last() *Pager { - return p.pagers[len(p.pagers)-1] -} - -// Pagers returns a list of pagers that can be used to build a pagination menu. -func (p *paginator) Pagers() pagers { - return p.pagers -} - -// PageSize returns the size of each paginator page. -func (p *paginator) PageSize() int { - return p.size -} - -// TotalPages returns the number of pages in the paginator. -func (p *paginator) TotalPages() int { - return len(p.paginatedElements) -} - -// TotalNumberOfElements returns the number of elements on all pages in this paginator. -func (p *paginator) TotalNumberOfElements() int { - return p.total -} - -func splitPages(pages Pages, size int) []paginatedElement { - var split []paginatedElement - for low, j := 0, len(pages); low < j; low += size { - high := int(math.Min(float64(low+size), float64(len(pages)))) - split = append(split, pages[low:high]) - } - - return split -} - -func splitPageGroups(pageGroups PagesGroup, size int) []paginatedElement { - - type keyPage struct { - key interface{} - page *Page - } - - var ( - split []paginatedElement - flattened []keyPage - ) - - for _, g := range pageGroups { - for _, p := range g.Pages { - flattened = append(flattened, keyPage{g.Key, p}) - } - } - - numPages := len(flattened) - - for low, j := 0, numPages; low < j; low += size { - high := int(math.Min(float64(low+size), float64(numPages))) - - var ( - pg PagesGroup - key interface{} - groupIndex = -1 - ) - - for k := low; k < high; k++ { - kp := flattened[k] - if key == nil || key != kp.key { - key = kp.key - pg = append(pg, PageGroup{Key: key}) - groupIndex++ - } - pg[groupIndex].Pages = append(pg[groupIndex].Pages, kp.page) - } - split = append(split, pg) - } - - return split -} - -// Paginator get this Page's main output's paginator. -func (p *Page) Paginator(options ...interface{}) (*Pager, error) { - return p.mainPageOutput.Paginator(options...) -} - -// Paginator gets this PageOutput's paginator if it's already created. -// If it's not, one will be created with all pages in Data["Pages"]. -func (p *PageOutput) Paginator(options ...interface{}) (*Pager, error) { - if !p.IsNode() { - return nil, fmt.Errorf("Paginators not supported for pages of type %q (%q)", p.Kind, p.title) - } - pagerSize, err := resolvePagerSize(p.s.Cfg, options...) - - if err != nil { - return nil, err - } - - var initError error - - p.paginatorInit.Do(func() { - if p.paginator != nil { - return - } - - pathDescriptor := p.targetPathDescriptor - if p.s.owner.IsMultihost() { - pathDescriptor.LangPrefix = "" - } - pagers, err := paginatePages(pathDescriptor, p.data["Pages"], pagerSize) - - if err != nil { - initError = err - } - - if len(pagers) > 0 { - // the rest of the nodes will be created later - p.paginator = pagers[0] - p.paginator.source = "paginator" - p.paginator.options = options - } - - }) - - if initError != nil { - return nil, initError - } - - return p.paginator, nil -} - -// Paginate invokes this Page's main output's Paginate method. -func (p *Page) Paginate(seq interface{}, options ...interface{}) (*Pager, error) { - return p.mainPageOutput.Paginate(seq, options...) -} - -// Paginate gets this PageOutput's paginator if it's already created. -// If it's not, one will be created with the qiven sequence. -// Note that repeated calls will return the same result, even if the sequence is different. -func (p *PageOutput) Paginate(seq interface{}, options ...interface{}) (*Pager, error) { - if !p.IsNode() { - return nil, fmt.Errorf("Paginators not supported for pages of type %q (%q)", p.Kind, p.title) - } - - pagerSize, err := resolvePagerSize(p.s.Cfg, options...) - - if err != nil { - return nil, err - } - - var initError error - - p.paginatorInit.Do(func() { - if p.paginator != nil { - return - } - - pathDescriptor := p.targetPathDescriptor - if p.s.owner.IsMultihost() { - pathDescriptor.LangPrefix = "" - } - pagers, err := paginatePages(pathDescriptor, seq, pagerSize) - - if err != nil { - initError = err - } - - if len(pagers) > 0 { - // the rest of the nodes will be created later - p.paginator = pagers[0] - p.paginator.source = seq - p.paginator.options = options - } - - }) - - if initError != nil { - return nil, initError - } - - if p.paginator.source == "paginator" { - return nil, errors.New("a Paginator was previously built for this Node without filters; look for earlier .Paginator usage") - } - - if !reflect.DeepEqual(options, p.paginator.options) || !probablyEqualPageLists(p.paginator.source, seq) { - return nil, errors.New("invoked multiple times with different arguments") - } - - return p.paginator, nil -} - -func resolvePagerSize(cfg config.Provider, options ...interface{}) (int, error) { - if len(options) == 0 { - return cfg.GetInt("paginate"), nil - } - - if len(options) > 1 { - return -1, errors.New("too many arguments, 'pager size' is currently the only option") - } - - pas, err := cast.ToIntE(options[0]) - - if err != nil || pas <= 0 { - return -1, errors.New(("'pager size' must be a positive integer")) - } - - return pas, nil -} - -func paginatePages(td targetPathDescriptor, seq interface{}, pagerSize int) (pagers, error) { - - if pagerSize <= 0 { - return nil, errors.New("'paginate' configuration setting must be positive to paginate") - } - - urlFactory := newPaginationURLFactory(td) - - var paginator *paginator - - groups, err := toPagesGroup(seq) - if err != nil { - return nil, err - } - if groups != nil { - paginator, _ = newPaginatorFromPageGroups(groups, pagerSize, urlFactory) - } else { - pages, err := toPages(seq) - if err != nil { - return nil, err - } - paginator, _ = newPaginatorFromPages(pages, pagerSize, urlFactory) - } - - pagers := paginator.Pagers() - - return pagers, nil -} - -func toPagesGroup(seq interface{}) (PagesGroup, error) { - switch v := seq.(type) { - case nil: - return nil, nil - case PagesGroup: - return v, nil - case []PageGroup: - return PagesGroup(v), nil - case []interface{}: - l := len(v) - if l == 0 { - break - } - switch v[0].(type) { - case PageGroup: - pagesGroup := make(PagesGroup, l) - for i, ipg := range v { - if pg, ok := ipg.(PageGroup); ok { - pagesGroup[i] = pg - } else { - return nil, fmt.Errorf("unsupported type in paginate from slice, got %T instead of PageGroup", ipg) - } - } - return PagesGroup(pagesGroup), nil - } - } - - return nil, nil -} - -func toPages(seq interface{}) (Pages, error) { - if seq == nil { - return Pages{}, nil - } - - switch v := seq.(type) { - case Pages: - return v, nil - case *Pages: - return *(v), nil - case []*Page: - return Pages(v), nil - case WeightedPages: - return v.Pages(), nil - case PageGroup: - return v.Pages, nil - case []interface{}: - pages := make(Pages, len(v)) - success := true - for i, vv := range v { - p, ok := vv.(*Page) - if !ok { - success = false - break - } - pages[i] = p - } - if success { - return pages, nil - } - } - - return nil, fmt.Errorf("cannot convert type %T to Pages", seq) -} - -// probablyEqual checks page lists for probable equality. -// It may return false positives. -// The motivation behind this is to avoid potential costly reflect.DeepEqual -// when "probably" is good enough. -func probablyEqualPageLists(a1 interface{}, a2 interface{}) bool { - - if a1 == nil || a2 == nil { - return a1 == a2 - } - - t1 := reflect.TypeOf(a1) - t2 := reflect.TypeOf(a2) - - if t1 != t2 { - return false - } - - if g1, ok := a1.(PagesGroup); ok { - g2 := a2.(PagesGroup) - if len(g1) != len(g2) { - return false - } - if len(g1) == 0 { - return true - } - if g1.Len() != g2.Len() { - return false - } - - return g1[0].Pages[0] == g2[0].Pages[0] - } - - p1, err1 := toPages(a1) - p2, err2 := toPages(a2) - - // probably the same wrong type - if err1 != nil && err2 != nil { - return true - } - - if len(p1) != len(p2) { - return false - } - - if len(p1) == 0 { - return true - } - - return p1[0] == p2[0] -} - -func newPaginatorFromPages(pages Pages, size int, urlFactory paginationURLFactory) (*paginator, error) { - - if size <= 0 { - return nil, errors.New("Paginator size must be positive") - } - - split := splitPages(pages, size) - - return newPaginator(split, len(pages), size, urlFactory) -} - -func newPaginatorFromPageGroups(pageGroups PagesGroup, size int, urlFactory paginationURLFactory) (*paginator, error) { - - if size <= 0 { - return nil, errors.New("Paginator size must be positive") - } - - split := splitPageGroups(pageGroups, size) - - return newPaginator(split, pageGroups.Len(), size, urlFactory) -} - -func newPaginator(elements []paginatedElement, total, size int, urlFactory paginationURLFactory) (*paginator, error) { - p := &paginator{total: total, paginatedElements: elements, size: size, paginationURLFactory: urlFactory} - - var ps pagers - - if len(elements) > 0 { - ps = make(pagers, len(elements)) - for i := range p.paginatedElements { - ps[i] = &Pager{number: (i + 1), paginator: p} - } - } else { - ps = make(pagers, 1) - ps[0] = &Pager{number: 1, paginator: p} - } - - p.pagers = ps - - return p, nil -} - -func newPaginationURLFactory(d targetPathDescriptor) paginationURLFactory { - - return func(page int) string { - pathDescriptor := d - var rel string - if page > 1 { - rel = fmt.Sprintf("/%s/%d/", d.PathSpec.PaginatePath, page) - pathDescriptor.Addends = rel - } - - targetPath := createTargetPath(pathDescriptor) - targetPath = strings.TrimSuffix(targetPath, d.Type.BaseFilename()) - link := d.PathSpec.PrependBasePath(targetPath, false) - // Note: The targetPath is massaged with MakePathSanitized - return d.PathSpec.URLizeFilename(link) - } -} diff --git a/hugolib/pagination_test.go b/hugolib/pagination_test.go deleted file mode 100644 index 5dbef609b..000000000 --- a/hugolib/pagination_test.go +++ /dev/null @@ -1,579 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "fmt" - "html/template" - "path/filepath" - "strings" - "testing" - - "github.com/gohugoio/hugo/deps" - "github.com/gohugoio/hugo/output" - "github.com/stretchr/testify/require" -) - -func TestSplitPages(t *testing.T) { - t.Parallel() - s := newTestSite(t) - - pages := createTestPages(s, 21) - chunks := splitPages(pages, 5) - require.Equal(t, 5, len(chunks)) - - for i := 0; i < 4; i++ { - require.Equal(t, 5, chunks[i].Len()) - } - - lastChunk := chunks[4] - require.Equal(t, 1, lastChunk.Len()) - -} - -func TestSplitPageGroups(t *testing.T) { - t.Parallel() - s := newTestSite(t) - pages := createTestPages(s, 21) - groups, _ := pages.GroupBy("Weight", "desc") - chunks := splitPageGroups(groups, 5) - require.Equal(t, 5, len(chunks)) - - firstChunk := chunks[0] - - // alternate weight 5 and 10 - if groups, ok := firstChunk.(PagesGroup); ok { - require.Equal(t, 5, groups.Len()) - for _, pg := range groups { - // first group 10 in weight - require.Equal(t, 10, pg.Key) - for _, p := range pg.Pages { - require.True(t, p.fuzzyWordCount%2 == 0) // magic test - } - } - } else { - t.Fatal("Excepted PageGroup") - } - - lastChunk := chunks[4] - - if groups, ok := lastChunk.(PagesGroup); ok { - require.Equal(t, 1, groups.Len()) - for _, pg := range groups { - // last should have 5 in weight - require.Equal(t, 5, pg.Key) - for _, p := range pg.Pages { - require.True(t, p.fuzzyWordCount%2 != 0) // magic test - } - } - } else { - t.Fatal("Excepted PageGroup") - } - -} - -func TestPager(t *testing.T) { - t.Parallel() - s := newTestSite(t) - pages := createTestPages(s, 21) - groups, _ := pages.GroupBy("Weight", "desc") - - urlFactory := func(page int) string { - return fmt.Sprintf("page/%d/", page) - } - - _, err := newPaginatorFromPages(pages, -1, urlFactory) - require.NotNil(t, err) - - _, err = newPaginatorFromPageGroups(groups, -1, urlFactory) - require.NotNil(t, err) - - pag, err := newPaginatorFromPages(pages, 5, urlFactory) - require.Nil(t, err) - doTestPages(t, pag) - first := pag.Pagers()[0].First() - require.Equal(t, "Pager 1", first.String()) - require.NotEmpty(t, first.Pages()) - require.Empty(t, first.PageGroups()) - - pag, err = newPaginatorFromPageGroups(groups, 5, urlFactory) - require.Nil(t, err) - doTestPages(t, pag) - first = pag.Pagers()[0].First() - require.NotEmpty(t, first.PageGroups()) - require.Empty(t, first.Pages()) - -} - -func doTestPages(t *testing.T, paginator *paginator) { - - paginatorPages := paginator.Pagers() - - require.Equal(t, 5, len(paginatorPages)) - require.Equal(t, 21, paginator.TotalNumberOfElements()) - require.Equal(t, 5, paginator.PageSize()) - require.Equal(t, 5, paginator.TotalPages()) - - first := paginatorPages[0] - require.Equal(t, template.HTML("page/1/"), first.URL()) - require.Equal(t, first, first.First()) - require.True(t, first.HasNext()) - require.Equal(t, paginatorPages[1], first.Next()) - require.False(t, first.HasPrev()) - require.Nil(t, first.Prev()) - require.Equal(t, 5, first.NumberOfElements()) - require.Equal(t, 1, first.PageNumber()) - - third := paginatorPages[2] - require.True(t, third.HasNext()) - require.True(t, third.HasPrev()) - require.Equal(t, paginatorPages[1], third.Prev()) - - last := paginatorPages[4] - require.Equal(t, template.HTML("page/5/"), last.URL()) - require.Equal(t, last, last.Last()) - require.False(t, last.HasNext()) - require.Nil(t, last.Next()) - require.True(t, last.HasPrev()) - require.Equal(t, 1, last.NumberOfElements()) - require.Equal(t, 5, last.PageNumber()) -} - -func TestPagerNoPages(t *testing.T) { - t.Parallel() - s := newTestSite(t) - pages := createTestPages(s, 0) - groups, _ := pages.GroupBy("Weight", "desc") - - urlFactory := func(page int) string { - return fmt.Sprintf("page/%d/", page) - } - - paginator, _ := newPaginatorFromPages(pages, 5, urlFactory) - doTestPagerNoPages(t, paginator) - - first := paginator.Pagers()[0].First() - require.Empty(t, first.PageGroups()) - require.Empty(t, first.Pages()) - - paginator, _ = newPaginatorFromPageGroups(groups, 5, urlFactory) - doTestPagerNoPages(t, paginator) - - first = paginator.Pagers()[0].First() - require.Empty(t, first.PageGroups()) - require.Empty(t, first.Pages()) - -} - -func doTestPagerNoPages(t *testing.T, paginator *paginator) { - paginatorPages := paginator.Pagers() - - require.Equal(t, 1, len(paginatorPages)) - require.Equal(t, 0, paginator.TotalNumberOfElements()) - require.Equal(t, 5, paginator.PageSize()) - require.Equal(t, 0, paginator.TotalPages()) - - // pageOne should be nothing but the first - pageOne := paginatorPages[0] - require.NotNil(t, pageOne.First()) - require.False(t, pageOne.HasNext()) - require.False(t, pageOne.HasPrev()) - require.Nil(t, pageOne.Next()) - require.Equal(t, 1, len(pageOne.Pagers())) - require.Equal(t, 0, pageOne.Pages().Len()) - require.Equal(t, 0, pageOne.NumberOfElements()) - require.Equal(t, 0, pageOne.TotalNumberOfElements()) - require.Equal(t, 0, pageOne.TotalPages()) - require.Equal(t, 1, pageOne.PageNumber()) - require.Equal(t, 5, pageOne.PageSize()) - -} - -func TestPaginationURLFactory(t *testing.T) { - t.Parallel() - cfg, fs := newTestCfg() - cfg.Set("paginatePath", "zoo") - - for _, uglyURLs := range []bool{false, true} { - for _, canonifyURLs := range []bool{false, true} { - t.Run(fmt.Sprintf("uglyURLs=%t,canonifyURLs=%t", uglyURLs, canonifyURLs), func(t *testing.T) { - - tests := []struct { - name string - d targetPathDescriptor - baseURL string - page int - expected string - }{ - {"HTML home page 32", - targetPathDescriptor{Kind: KindHome, Type: output.HTMLFormat}, "http://example.com/", 32, "/zoo/32/"}, - {"JSON home page 42", - targetPathDescriptor{Kind: KindHome, Type: output.JSONFormat}, "http://example.com/", 42, "/zoo/42/"}, - // Issue #1252 - {"BaseURL with sub path", - targetPathDescriptor{Kind: KindHome, Type: output.HTMLFormat}, "http://example.com/sub/", 999, "/sub/zoo/999/"}, - } - - for _, test := range tests { - d := test.d - cfg.Set("baseURL", test.baseURL) - cfg.Set("canonifyURLs", canonifyURLs) - cfg.Set("uglyURLs", uglyURLs) - d.UglyURLs = uglyURLs - - expected := test.expected - - if canonifyURLs { - expected = strings.Replace(expected, "/sub", "", 1) - } - - if uglyURLs { - expected = expected[:len(expected)-1] + "." + test.d.Type.MediaType.Suffix() - } - - pathSpec := newTestPathSpec(fs, cfg) - d.PathSpec = pathSpec - - factory := newPaginationURLFactory(d) - - got := factory(test.page) - - require.Equal(t, expected, got) - - } - }) - } - } -} - -func TestPaginator(t *testing.T) { - t.Parallel() - for _, useViper := range []bool{false, true} { - doTestPaginator(t, useViper) - } -} - -func doTestPaginator(t *testing.T, useViper bool) { - - cfg, fs := newTestCfg() - - pagerSize := 5 - if useViper { - cfg.Set("paginate", pagerSize) - } else { - cfg.Set("paginate", -1) - } - - s, err := NewSiteForCfg(deps.DepsCfg{Cfg: cfg, Fs: fs}) - require.NoError(t, err) - - pages := createTestPages(s, 12) - n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) - n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) - n1.data["Pages"] = pages - - var paginator1 *Pager - - if useViper { - paginator1, err = n1.Paginator() - } else { - paginator1, err = n1.Paginator(pagerSize) - } - - require.Nil(t, err) - require.NotNil(t, paginator1) - require.Equal(t, 3, paginator1.TotalPages()) - require.Equal(t, 12, paginator1.TotalNumberOfElements()) - - n2.paginator = paginator1.Next() - paginator2, err := n2.Paginator() - require.Nil(t, err) - require.Equal(t, paginator2, paginator1.Next()) - - n1.data["Pages"] = createTestPages(s, 1) - samePaginator, _ := n1.Paginator() - require.Equal(t, paginator1, samePaginator) - - pp, _ := s.NewPage("test") - p, _ := newPageOutput(pp, false, false, output.HTMLFormat) - - _, err = p.Paginator() - require.NotNil(t, err) -} - -func TestPaginatorWithNegativePaginate(t *testing.T) { - t.Parallel() - s := newTestSite(t, "paginate", -1) - n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) - _, err := n1.Paginator() - require.Error(t, err) -} - -func TestPaginate(t *testing.T) { - t.Parallel() - for _, useViper := range []bool{false, true} { - doTestPaginate(t, useViper) - } -} - -func TestPaginatorURL(t *testing.T) { - t.Parallel() - cfg, fs := newTestCfg() - - cfg.Set("paginate", 2) - cfg.Set("paginatePath", "testing") - - for i := 0; i < 10; i++ { - // Issue #2177, do not double encode URLs - writeSource(t, fs, filepath.Join("content", "阅读", fmt.Sprintf("page%d.md", (i+1))), - fmt.Sprintf(`--- -title: Page%d ---- -Conten%d -`, (i+1), i+1)) - - } - writeSource(t, fs, filepath.Join("layouts", "_default", "single.html"), "<html><body>{{.Content}}</body></html>") - writeSource(t, fs, filepath.Join("layouts", "_default", "list.html"), - ` -<html><body> -Count: {{ .Paginator.TotalNumberOfElements }} -Pages: {{ .Paginator.TotalPages }} -{{ range .Paginator.Pagers -}} - {{ .PageNumber }}: {{ .URL }} -{{ end }} -</body></html>`) - - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - - th := testHelper{s.Cfg, s.Fs, t} - - th.assertFileContent(filepath.Join("public", "阅读", "testing", "2", "index.html"), "2: /%E9%98%85%E8%AF%BB/testing/2/") - -} - -func doTestPaginate(t *testing.T, useViper bool) { - pagerSize := 5 - - var ( - s *Site - err error - ) - - if useViper { - s = newTestSite(t, "paginate", pagerSize) - } else { - s = newTestSite(t, "paginate", -1) - } - - pages := createTestPages(s, 6) - n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) - n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) - - var paginator1, paginator2 *Pager - - if useViper { - paginator1, err = n1.Paginate(pages) - } else { - paginator1, err = n1.Paginate(pages, pagerSize) - } - - require.Nil(t, err) - require.NotNil(t, paginator1) - require.Equal(t, 2, paginator1.TotalPages()) - require.Equal(t, 6, paginator1.TotalNumberOfElements()) - - n2.paginator = paginator1.Next() - if useViper { - paginator2, err = n2.Paginate(pages) - } else { - paginator2, err = n2.Paginate(pages, pagerSize) - } - require.Nil(t, err) - require.Equal(t, paginator2, paginator1.Next()) - - pp, err := s.NewPage("test") - p, _ := newPageOutput(pp, false, false, output.HTMLFormat) - - _, err = p.Paginate(pages) - require.NotNil(t, err) -} - -func TestInvalidOptions(t *testing.T) { - t.Parallel() - s := newTestSite(t) - n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) - - _, err := n1.Paginate(createTestPages(s, 1), 1, 2) - require.NotNil(t, err) - _, err = n1.Paginator(1, 2) - require.NotNil(t, err) - _, err = n1.Paginator(-1) - require.NotNil(t, err) -} - -func TestPaginateWithNegativePaginate(t *testing.T) { - t.Parallel() - cfg, fs := newTestCfg() - cfg.Set("paginate", -1) - - s, err := NewSiteForCfg(deps.DepsCfg{Cfg: cfg, Fs: fs}) - require.NoError(t, err) - - n, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) - - _, err = n.Paginate(createTestPages(s, 2)) - require.NotNil(t, err) -} - -func TestPaginatePages(t *testing.T) { - t.Parallel() - s := newTestSite(t) - - groups, _ := createTestPages(s, 31).GroupBy("Weight", "desc") - pd := targetPathDescriptor{Kind: KindHome, Type: output.HTMLFormat, PathSpec: s.PathSpec, Addends: "t"} - - for i, seq := range []interface{}{createTestPages(s, 11), groups, WeightedPages{}, PageGroup{}, &Pages{}} { - v, err := paginatePages(pd, seq, 11) - require.NotNil(t, v, "Val %d", i) - require.Nil(t, err, "Err %d", i) - } - _, err := paginatePages(pd, Site{}, 11) - require.NotNil(t, err) - -} - -// Issue #993 -func TestPaginatorFollowedByPaginateShouldFail(t *testing.T) { - t.Parallel() - s := newTestSite(t, "paginate", 10) - n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) - n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) - - _, err := n1.Paginator() - require.Nil(t, err) - _, err = n1.Paginate(createTestPages(s, 2)) - require.NotNil(t, err) - - _, err = n2.Paginate(createTestPages(s, 2)) - require.Nil(t, err) - -} - -func TestPaginateFollowedByDifferentPaginateShouldFail(t *testing.T) { - t.Parallel() - s := newTestSite(t, "paginate", 10) - - n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) - n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) - - p1 := createTestPages(s, 2) - p2 := createTestPages(s, 10) - - _, err := n1.Paginate(p1) - require.Nil(t, err) - - _, err = n1.Paginate(p1) - require.Nil(t, err) - - _, err = n1.Paginate(p2) - require.NotNil(t, err) - - _, err = n2.Paginate(p2) - require.Nil(t, err) -} - -func TestProbablyEqualPageLists(t *testing.T) { - t.Parallel() - s := newTestSite(t) - fivePages := createTestPages(s, 5) - zeroPages := createTestPages(s, 0) - zeroPagesByWeight, _ := createTestPages(s, 0).GroupBy("Weight", "asc") - fivePagesByWeight, _ := createTestPages(s, 5).GroupBy("Weight", "asc") - ninePagesByWeight, _ := createTestPages(s, 9).GroupBy("Weight", "asc") - - for i, this := range []struct { - v1 interface{} - v2 interface{} - expect bool - }{ - {nil, nil, true}, - {"a", "b", true}, - {"a", fivePages, false}, - {fivePages, "a", false}, - {fivePages, createTestPages(s, 2), false}, - {fivePages, fivePages, true}, - {zeroPages, zeroPages, true}, - {fivePagesByWeight, fivePagesByWeight, true}, - {zeroPagesByWeight, fivePagesByWeight, false}, - {zeroPagesByWeight, zeroPagesByWeight, true}, - {fivePagesByWeight, fivePages, false}, - {fivePagesByWeight, ninePagesByWeight, false}, - } { - result := probablyEqualPageLists(this.v1, this.v2) - - if result != this.expect { - t.Errorf("[%d] got %t but expected %t", i, result, this.expect) - - } - } -} - -func TestPage(t *testing.T) { - t.Parallel() - urlFactory := func(page int) string { - return fmt.Sprintf("page/%d/", page) - } - - s := newTestSite(t) - - fivePages := createTestPages(s, 7) - fivePagesFuzzyWordCount, _ := createTestPages(s, 7).GroupBy("FuzzyWordCount", "asc") - - p1, _ := newPaginatorFromPages(fivePages, 2, urlFactory) - p2, _ := newPaginatorFromPageGroups(fivePagesFuzzyWordCount, 2, urlFactory) - - f1 := p1.pagers[0].First() - f2 := p2.pagers[0].First() - - page11, _ := f1.page(1) - page1Nil, _ := f1.page(3) - - page21, _ := f2.page(1) - page2Nil, _ := f2.page(3) - - require.Equal(t, 3, page11.fuzzyWordCount) - require.Nil(t, page1Nil) - - require.Equal(t, 3, page21.fuzzyWordCount) - require.Nil(t, page2Nil) -} - -func createTestPages(s *Site, num int) Pages { - pages := make(Pages, num) - - for i := 0; i < num; i++ { - p := s.newPage(filepath.FromSlash(fmt.Sprintf("/x/y/z/p%d.md", i))) - w := 5 - if i%2 == 0 { - w = 10 - } - p.fuzzyWordCount = i + 2 - p.Weight = w - pages[i] = p - - } - - return pages -} diff --git a/hugolib/path_separators_test.go b/hugolib/path_separators_test.go deleted file mode 100644 index 0d769e650..000000000 --- a/hugolib/path_separators_test.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "path/filepath" - "strings" - "testing" -) - -var simplePageYAML = `--- -contenttype: "" ---- -Sample Text -` - -func TestDegenerateMissingFolderInPageFilename(t *testing.T) { - t.Parallel() - s := newTestSite(t) - p, err := s.newPageFrom(strings.NewReader(simplePageYAML), filepath.Join("foobar")) - if err != nil { - t.Fatalf("Error in NewPageFrom") - } - if p.Section() != "" { - t.Fatalf("No section should be set for a file path: foobar") - } -} diff --git a/hugolib/paths/themes.go b/hugolib/paths/themes.go index 4718720e1..a526953f1 100644 --- a/hugolib/paths/themes.go +++ b/hugolib/paths/themes.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -75,7 +75,7 @@ func (c *themesCollector) add(name, configFilename string) (ThemeConfig, error) var err error cfg, err = config.FromFile(c.fs, configFilename) if err != nil { - return tc, nil + return tc, err } } diff --git a/hugolib/permalinker.go b/hugolib/permalinker.go index 5e7a13a02..29dad6ce4 100644 --- a/hugolib/permalinker.go +++ b/hugolib/permalinker.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,8 +14,7 @@ package hugolib var ( - _ Permalinker = (*Page)(nil) - _ Permalinker = (*OutputFormat)(nil) + _ Permalinker = (*pageState)(nil) ) // Permalinker provides permalinks of both the relative and absolute kind. diff --git a/hugolib/permalinks.go b/hugolib/permalinks.go deleted file mode 100644 index 3d261a113..000000000 --- a/hugolib/permalinks.go +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "errors" - "fmt" - "path" - "path/filepath" - "regexp" - "strconv" - "strings" - - "github.com/gohugoio/hugo/helpers" -) - -// pathPattern represents a string which builds up a URL from attributes -type pathPattern string - -// pageToPermaAttribute is the type of a function which, given a page and a tag -// can return a string to go in that position in the page (or an error) -type pageToPermaAttribute func(*Page, string) (string, error) - -// PermalinkOverrides maps a section name to a PathPattern -type PermalinkOverrides map[string]pathPattern - -// knownPermalinkAttributes maps :tags in a permalink specification to a -// function which, given a page and the tag, returns the resulting string -// to be used to replace that tag. -var knownPermalinkAttributes map[string]pageToPermaAttribute - -var attributeRegexp = regexp.MustCompile(`:\w+`) - -// validate determines if a PathPattern is well-formed -func (pp pathPattern) validate() bool { - fragments := strings.Split(string(pp[1:]), "/") - var bail = false - for i := range fragments { - if bail { - return false - } - if len(fragments[i]) == 0 { - bail = true - continue - } - - matches := attributeRegexp.FindAllStringSubmatch(fragments[i], -1) - if matches == nil { - continue - } - - for _, match := range matches { - k := strings.ToLower(match[0][1:]) - if _, ok := knownPermalinkAttributes[k]; !ok { - return false - } - } - } - return true -} - -type permalinkExpandError struct { - pattern pathPattern - section string - err error -} - -func (pee *permalinkExpandError) Error() string { - return fmt.Sprintf("error expanding %q section %q: %s", string(pee.pattern), pee.section, pee.err) -} - -var ( - errPermalinkIllFormed = errors.New("permalink ill-formed") - errPermalinkAttributeUnknown = errors.New("permalink attribute not recognised") -) - -// Expand on a PathPattern takes a Page and returns the fully expanded Permalink -// or an error explaining the failure. -func (pp pathPattern) Expand(p *Page) (string, error) { - if !pp.validate() { - return "", &permalinkExpandError{pattern: pp, section: "<all>", err: errPermalinkIllFormed} - } - sections := strings.Split(string(pp), "/") - for i, field := range sections { - if len(field) == 0 { - continue - } - - matches := attributeRegexp.FindAllStringSubmatch(field, -1) - - if matches == nil { - continue - } - - newField := field - - for _, match := range matches { - attr := match[0][1:] - callback, ok := knownPermalinkAttributes[attr] - - if !ok { - return "", &permalinkExpandError{pattern: pp, section: strconv.Itoa(i), err: errPermalinkAttributeUnknown} - } - - newAttr, err := callback(p, attr) - - if err != nil { - return "", &permalinkExpandError{pattern: pp, section: strconv.Itoa(i), err: err} - } - - newField = strings.Replace(newField, match[0], newAttr, 1) - } - - sections[i] = newField - } - return strings.Join(sections, "/"), nil -} - -func pageToPermalinkDate(p *Page, dateField string) (string, error) { - // a Page contains a Node which provides a field Date, time.Time - switch dateField { - case "year": - return strconv.Itoa(p.Date.Year()), nil - case "month": - return fmt.Sprintf("%02d", int(p.Date.Month())), nil - case "monthname": - return p.Date.Month().String(), nil - case "day": - return fmt.Sprintf("%02d", p.Date.Day()), nil - case "weekday": - return strconv.Itoa(int(p.Date.Weekday())), nil - case "weekdayname": - return p.Date.Weekday().String(), nil - case "yearday": - return strconv.Itoa(p.Date.YearDay()), nil - } - //TODO: support classic strftime escapes too - // (and pass those through despite not being in the map) - panic("coding error: should not be here") -} - -// pageToPermalinkTitle returns the URL-safe form of the title -func pageToPermalinkTitle(p *Page, _ string) (string, error) { - // Page contains Node which has Title - // (also contains URLPath which has Slug, sometimes) - return p.s.PathSpec.URLize(p.title), nil -} - -// pageToPermalinkFilename returns the URL-safe form of the filename -func pageToPermalinkFilename(p *Page, _ string) (string, error) { - name := p.File.TranslationBaseName() - if name == "index" { - // Page bundles; the directory name will hopefully have a better name. - dir := strings.TrimSuffix(p.File.Dir(), helpers.FilePathSeparator) - _, name = filepath.Split(dir) - } - - return p.s.PathSpec.URLize(name), nil -} - -// if the page has a slug, return the slug, else return the title -func pageToPermalinkSlugElseTitle(p *Page, a string) (string, error) { - if p.Slug != "" { - // Don't start or end with a - - // TODO(bep) this doesn't look good... Set the Slug once. - if strings.HasPrefix(p.Slug, "-") { - p.Slug = p.Slug[1:len(p.Slug)] - } - - if strings.HasSuffix(p.Slug, "-") { - p.Slug = p.Slug[0 : len(p.Slug)-1] - } - return p.s.PathSpec.URLize(p.Slug), nil - } - return pageToPermalinkTitle(p, a) -} - -func pageToPermalinkSection(p *Page, _ string) (string, error) { - return p.Section(), nil -} - -func pageToPermalinkSections(p *Page, _ string) (string, error) { - return path.Join(p.CurrentSection().sections...), nil -} - -func init() { - knownPermalinkAttributes = map[string]pageToPermaAttribute{ - "year": pageToPermalinkDate, - "month": pageToPermalinkDate, - "monthname": pageToPermalinkDate, - "day": pageToPermalinkDate, - "weekday": pageToPermalinkDate, - "weekdayname": pageToPermalinkDate, - "yearday": pageToPermalinkDate, - "section": pageToPermalinkSection, - "sections": pageToPermalinkSections, - "title": pageToPermalinkTitle, - "slug": pageToPermalinkSlugElseTitle, - "filename": pageToPermalinkFilename, - } - -} diff --git a/hugolib/permalinks_test.go b/hugolib/permalinks_test.go deleted file mode 100644 index 7bc242955..000000000 --- a/hugolib/permalinks_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "path/filepath" - "strings" - "testing" -) - -// testdataPermalinks is used by a couple of tests; the expandsTo content is -// subject to the data in simplePageJSON. -var testdataPermalinks = []struct { - spec string - valid bool - expandsTo string -}{ - {":title", true, "spf13-vim-3.0-release-and-new-website"}, - {"/:year-:month-:title", true, "/2012-04-spf13-vim-3.0-release-and-new-website"}, - - {"/:year/:yearday/:month/:monthname/:day/:weekday/:weekdayname/", true, "/2012/97/04/April/06/5/Friday/"}, // Dates - {"/:section/", true, "/blue/"}, // Section - {"/:title/", true, "/spf13-vim-3.0-release-and-new-website/"}, // Title - {"/:slug/", true, "/spf13-vim-3-0-release-and-new-website/"}, // Slug - {"/:filename/", true, "/test-page/"}, // Filename - // TODO(moorereason): need test scaffolding for this. - //{"/:sections/", false, "/blue/"}, // Sections - - // Failures - {"/blog/:fred", false, ""}, - {"/:year//:title", false, ""}, -} - -func TestPermalinkValidation(t *testing.T) { - t.Parallel() - for _, item := range testdataPermalinks { - pp := pathPattern(item.spec) - have := pp.validate() - if have == item.valid { - continue - } - var howBad string - if have { - howBad = "validates but should not have" - } else { - howBad = "should have validated but did not" - } - t.Errorf("permlink spec %q %s", item.spec, howBad) - } -} - -func TestPermalinkExpansion(t *testing.T) { - t.Parallel() - s := newTestSite(t) - page, err := s.newPageFrom(strings.NewReader(simplePageJSON), filepath.FromSlash("blue/test-page.md")) - - if err != nil { - t.Fatalf("failed before we began, could not parse simplePageJSON: %s", err) - } - for _, item := range testdataPermalinks { - if !item.valid { - continue - } - pp := pathPattern(item.spec) - result, err := pp.Expand(page) - if err != nil { - t.Errorf("failed to expand page: %s", err) - continue - } - if result != item.expandsTo { - t.Errorf("expansion mismatch!\n\tExpected: %q\n\tReceived: %q", item.expandsTo, result) - } - } -} diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index f53ab4966..e22121b77 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -39,7 +39,7 @@ func TestSCSSWithIncludePaths(t *testing.T) { v := viper.New() v.Set("workingDir", workDir) - b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger()) + b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) b.WithViper(v) b.WithWorkingDir(workDir) // Need to use OS fs for this. @@ -94,7 +94,7 @@ func TestSCSSWithThemeOverrides(t *testing.T) { v := viper.New() v.Set("workingDir", workDir) v.Set("theme", theme) - b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger()) + b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) b.WithViper(v) b.WithWorkingDir(workDir) // Need to use OS fs for this. @@ -367,7 +367,7 @@ CSV2: {{ $csv2 }} continue } - b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger()) + b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) b.WithSimpleConfigFile() b.WithContent("_index.md", ` --- diff --git a/hugolib/rss_test.go b/hugolib/rss_test.go index db26c7d2d..683a737c5 100644 --- a/hugolib/rss_test.go +++ b/hugolib/rss_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -47,7 +47,7 @@ func TestRSSOutput(t *testing.T) { // Section RSS th.assertFileContent(filepath.Join("public", "sect", rssURI), "<?xml", "rss version", "Sects on RSSTest") // Taxonomy RSS - th.assertFileContent(filepath.Join("public", "categories", "hugo", rssURI), "<?xml", "rss version", "Hugo on RSSTest") + th.assertFileContent(filepath.Join("public", "categories", "hugo", rssURI), "<?xml", "rss version", "hugo on RSSTest") // RSS Item Limit content := readDestination(t, fs, filepath.Join("public", rssURI)) @@ -74,3 +74,24 @@ func TestRSSKind(t *testing.T) { b.AssertFileContent("public/index.xml", "RSS Kind: home") } + +func TestRSSCanonifyURLs(t *testing.T) { + t.Parallel() + + b := newTestSitesBuilder(t) + b.WithSimpleConfigFile().WithTemplatesAdded("index.rss.xml", `<rss>{{ range .Pages }}<item>{{ .Content | html }}</item>{{ end }}</rss>`) + b.WithContent("page.md", `--- +Title: My Page +--- + +Figure: + +{{< figure src="/images/sunset.jpg" title="Sunset" >}} + + + +`) + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.xml", "img src="http://example.com/images/sunset.jpg") +} diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index cd2f268f1..68455d30f 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,12 +15,14 @@ package hugolib import ( "bytes" - "errors" "fmt" + "strconv" + "html/template" "path" "github.com/gohugoio/hugo/common/herrors" + "github.com/pkg/errors" "reflect" @@ -28,6 +30,7 @@ import ( "sort" "github.com/gohugoio/hugo/parser/pageparser" + "github.com/gohugoio/hugo/resources/page" _errors "github.com/pkg/errors" @@ -39,8 +42,6 @@ import ( "github.com/gohugoio/hugo/common/urls" "github.com/gohugoio/hugo/output" - "github.com/gohugoio/hugo/media" - bp "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/tpl" @@ -48,7 +49,7 @@ import ( var ( _ urls.RefLinker = (*ShortcodeWithPage)(nil) - _ pageContainer = (*ShortcodeWithPage)(nil) + _ pageWrapper = (*ShortcodeWithPage)(nil) _ text.Positioner = (*ShortcodeWithPage)(nil) ) @@ -56,7 +57,7 @@ var ( type ShortcodeWithPage struct { Params interface{} Inner template.HTML - Page *PageWithoutContent + Page page.Page Parent *ShortcodeWithPage Name string IsNamedParams bool @@ -77,26 +78,28 @@ type ShortcodeWithPage struct { // may be expensive to calculate, so only use this in error situations. func (scp *ShortcodeWithPage) Position() text.Position { scp.posInit.Do(func() { - scp.pos = scp.Page.posFromPage(scp.posOffset) + if p, ok := mustUnwrapPage(scp.Page).(pageContext); ok { + scp.pos = p.posOffset(scp.posOffset) + } }) return scp.pos } // Site returns information about the current site. -func (scp *ShortcodeWithPage) Site() *SiteInfo { - return scp.Page.Site +func (scp *ShortcodeWithPage) Site() page.Site { + return scp.Page.Site() } // Ref is a shortcut to the Ref method on Page. It passes itself as a context // to get better error messages. func (scp *ShortcodeWithPage) Ref(args map[string]interface{}) (string, error) { - return scp.Page.ref(args, scp) + return scp.Page.RefFrom(args, scp) } // RelRef is a shortcut to the RelRef method on Page. It passes itself as a context // to get better error messages. func (scp *ShortcodeWithPage) RelRef(args map[string]interface{}) (string, error) { - return scp.Page.relRef(args, scp) + return scp.Page.RelRefFrom(args, scp) } // Scratch returns a scratch-pad scoped for this shortcode. This can be used @@ -159,12 +162,16 @@ func (scp *ShortcodeWithPage) Get(key interface{}) interface{} { } -func (scp *ShortcodeWithPage) page() *Page { - return scp.Page.Page +func (scp *ShortcodeWithPage) page() page.Page { + return scp.Page } // Note - this value must not contain any markup syntax -const shortcodePlaceholderPrefix = "HUGOSHORTCODE" +const shortcodePlaceholderPrefix = "HAHAHUGOSHORTCODE" + +func createShortcodePlaceholder(id string, ordinal int) string { + return shortcodePlaceholderPrefix + "-" + id + strconv.Itoa(ordinal) + "-HBHB" +} type shortcode struct { name string @@ -174,8 +181,24 @@ type shortcode struct { params interface{} // map or array ordinal int err error - doMarkup bool - pos int // the position in bytes in the source file + + info tpl.Info + + // If set, the rendered shortcode is sent as part of the surrounding content + // to Blackfriday and similar. + // Before Hug0 0.55 we didn't send any shortcode output to the markup + // renderer, and this flag told Hugo to process the {{ .Inner }} content + // separately. + // The old behaviour can be had by starting your shortcode template with: + // {{ $_hugo_config := `{ "version": 1 }`}} + doMarkup bool + + // the placeholder in the source when passed to Blackfriday etc. + // This also identifies the rendered shortcode. + placeholder string + + pos int // the position in bytes in the source file + length int // the length in bytes in the source file } func (s shortcode) innerString() string { @@ -214,193 +237,92 @@ func (sc shortcode) String() string { return fmt.Sprintf("%s(%q, %t){%s}", sc.name, params, sc.doMarkup, sc.inner) } -// We may have special shortcode templates for AMP etc. -// Note that in the below, OutputFormat may be empty. -// We will try to look for the most specific shortcode template available. -type scKey struct { - Lang string - OutputFormat string - Suffix string - ShortcodePlaceholder string -} - -func newScKey(m media.Type, shortcodeplaceholder string) scKey { - return scKey{Suffix: m.Suffix(), ShortcodePlaceholder: shortcodeplaceholder} -} - -func newScKeyFromLangAndOutputFormat(lang string, o output.Format, shortcodeplaceholder string) scKey { - return scKey{Lang: lang, Suffix: o.MediaType.Suffix(), OutputFormat: o.Name, ShortcodePlaceholder: shortcodeplaceholder} -} - -func newDefaultScKey(shortcodeplaceholder string) scKey { - return newScKey(media.HTMLType, shortcodeplaceholder) -} - type shortcodeHandler struct { - init sync.Once - - p *PageWithoutContent - - // This is all shortcode rendering funcs for all potential output formats. - contentShortcodes *orderedMap + p *pageState - // This map contains the new or changed set of shortcodes that need - // to be rendered for the current output format. - contentShortcodesDelta *orderedMap + s *Site - // This maps the shorcode placeholders with the rendered content. - // We will do (potential) partial re-rendering per output format, - // so keep this for the unchanged. - renderedShortcodes map[string]string - - // Maps the shortcodeplaceholder with the actual shortcode. - shortcodes *orderedMap + // Ordered list of shortcodes for a page. + shortcodes []*shortcode // All the shortcode names in this set. nameSet map[string]bool - placeholderID int - placeholderFunc func() string - + // Configuration enableInlineShortcodes bool } -func (s *shortcodeHandler) nextPlaceholderID() int { - s.placeholderID++ - return s.placeholderID -} +func newShortcodeHandler(p *pageState, s *Site, placeholderFunc func() string) *shortcodeHandler { -func (s *shortcodeHandler) createShortcodePlaceholder() string { - return s.placeholderFunc() -} - -func newShortcodeHandler(p *Page) *shortcodeHandler { - - s := &shortcodeHandler{ - p: p.withoutContent(), - enableInlineShortcodes: p.s.enableInlineShortcodes, - contentShortcodes: newOrderedMap(), - shortcodes: newOrderedMap(), + sh := &shortcodeHandler{ + p: p, + s: s, + enableInlineShortcodes: s.enableInlineShortcodes, + shortcodes: make([]*shortcode, 0, 4), nameSet: make(map[string]bool), - renderedShortcodes: make(map[string]string), - } - - placeholderFunc := p.s.shortcodePlaceholderFunc - if placeholderFunc == nil { - placeholderFunc = func() string { - return fmt.Sprintf("HAHA%s-%p-%d-HBHB", shortcodePlaceholderPrefix, p, s.nextPlaceholderID()) - } - - } - s.placeholderFunc = placeholderFunc - return s -} - -// TODO(bep) make it non-global -var isInnerShortcodeCache = struct { - sync.RWMutex - m map[string]bool -}{m: make(map[string]bool)} - -// to avoid potential costly look-aheads for closing tags we look inside the template itself -// we could change the syntax to self-closing tags, but that would make users cry -// the value found is cached -func isInnerShortcode(t tpl.TemplateExecutor) (bool, error) { - isInnerShortcodeCache.RLock() - m, ok := isInnerShortcodeCache.m[t.Name()] - isInnerShortcodeCache.RUnlock() - - if ok { - return m, nil } - isInnerShortcodeCache.Lock() - defer isInnerShortcodeCache.Unlock() - match, _ := regexp.MatchString("{{.*?\\.Inner.*?}}", t.Tree()) - isInnerShortcodeCache.m[t.Name()] = match - - return match, nil -} - -func clearIsInnerShortcodeCache() { - isInnerShortcodeCache.Lock() - defer isInnerShortcodeCache.Unlock() - isInnerShortcodeCache.m = make(map[string]bool) + return sh } -const innerNewlineRegexp = "\n" -const innerCleanupRegexp = `\A<p>(.*)</p>\n\z` -const innerCleanupExpand = "$1" - -func (s *shortcodeHandler) prepareShortcodeForPage(placeholder string, sc *shortcode, parent *ShortcodeWithPage, p *PageWithoutContent) map[scKey]func() (string, error) { - m := make(map[scKey]func() (string, error)) - lang := p.Lang() - - if sc.isInline { - key := newScKeyFromLangAndOutputFormat(lang, p.outputFormats[0], placeholder) - m[key] = func() (string, error) { - return renderShortcode(key, sc, nil, p) - - } - - return m - - } - - for _, f := range p.outputFormats { - // The most specific template will win. - key := newScKeyFromLangAndOutputFormat(lang, f, placeholder) - m[key] = func() (string, error) { - return renderShortcode(key, sc, nil, p) - } - } - - return m -} +const ( + innerNewlineRegexp = "\n" + innerCleanupRegexp = `\A<p>(.*)</p>\n\z` + innerCleanupExpand = "$1" +) func renderShortcode( - tmplKey scKey, + level int, + s *Site, + tplVariants tpl.TemplateVariants, sc *shortcode, parent *ShortcodeWithPage, - p *PageWithoutContent) (string, error) { + p *pageState) (string, bool, error) { var tmpl tpl.Template + // Tracks whether this shortcode or any of its children has template variations + // in other languages or output formats. We are currently only interested in + // the output formats, so we may get some false positives -- we + // should improve on that. + var hasVariants bool + if sc.isInline { if !p.s.enableInlineShortcodes { - return "", nil + return "", false, nil } - templName := path.Join("_inline_shortcode", p.Path(), sc.name) + templName := path.Join("_inline_shortcode", p.File().Path(), sc.name) if sc.isClosing { templStr := sc.innerString() var err error - tmpl, err = p.s.TextTmpl.Parse(templName, templStr) + tmpl, err = s.TextTmpl.Parse(templName, templStr) if err != nil { fe := herrors.ToFileError("html", err) - l1, l2 := p.posFromPage(sc.pos).LineNumber, fe.Position().LineNumber + l1, l2 := p.posOffset(sc.pos).LineNumber, fe.Position().LineNumber fe = herrors.ToFileErrorWithLineNumber(fe, l1+l2-1) - return "", p.errWithFileContext(fe) + return "", false, p.wrapError(fe) } } else { // Re-use of shortcode defined earlier in the same page. var found bool - tmpl, found = p.s.TextTmpl.Lookup(templName) + tmpl, found = s.TextTmpl.Lookup(templName) if !found { - return "", _errors.Errorf("no earlier definition of shortcode %q found", sc.name) + return "", false, _errors.Errorf("no earlier definition of shortcode %q found", sc.name) } } } else { - tmpl = getShortcodeTemplateForTemplateKey(tmplKey, sc.name, p.s.Tmpl) - } - - if tmpl == nil { - p.s.Log.ERROR.Printf("Unable to locate template for shortcode %q in page %q", sc.name, p.Path()) - return "", nil + var found, more bool + tmpl, found, more = s.Tmpl.LookupVariant(sc.name, tplVariants) + if !found { + s.Log.ERROR.Printf("Unable to locate template for shortcode %q in page %q", sc.name, p.File().Path()) + return "", false, nil + } + hasVariants = hasVariants || more } - data := &ShortcodeWithPage{Ordinal: sc.ordinal, posOffset: sc.pos, Params: sc.params, Page: p, Parent: parent, Name: sc.name} + data := &ShortcodeWithPage{Ordinal: sc.ordinal, posOffset: sc.pos, Params: sc.params, Page: newPageForShortcode(p), Parent: parent, Name: sc.name} if sc.params != nil { data.IsNamedParams = reflect.TypeOf(sc.params).Kind() == reflect.Map } @@ -408,32 +330,35 @@ func renderShortcode( if len(sc.inner) > 0 { var inner string for _, innerData := range sc.inner { - switch innerData.(type) { + switch innerData := innerData.(type) { case string: - inner += innerData.(string) + inner += innerData case *shortcode: - s, err := renderShortcode(tmplKey, innerData.(*shortcode), data, p) + s, more, err := renderShortcode(level+1, s, tplVariants, innerData, data, p) if err != nil { - return "", err + return "", false, err } + hasVariants = hasVariants || more inner += s default: - p.s.Log.ERROR.Printf("Illegal state on shortcode rendering of %q in page %q. Illegal type in inner data: %s ", - sc.name, p.Path(), reflect.TypeOf(innerData)) - return "", nil + s.Log.ERROR.Printf("Illegal state on shortcode rendering of %q in page %q. Illegal type in inner data: %s ", + sc.name, p.File().Path(), reflect.TypeOf(innerData)) + return "", false, nil } } - if sc.doMarkup { - newInner := p.s.ContentSpec.RenderBytes(&helpers.RenderingContext{ + // Pre Hugo 0.55 this was the behaviour even for the outer-most + // shortcode. + if sc.doMarkup && (level > 0 || sc.info.Config.Version == 1) { + newInner := s.ContentSpec.RenderBytes(&helpers.RenderingContext{ Content: []byte(inner), - PageFmt: p.Markup, + PageFmt: p.m.markup, Cfg: p.Language(), - DocumentID: p.UniqueID(), - DocumentName: p.Path(), + DocumentID: p.File().UniqueID(), + DocumentName: p.File().Path(), Config: p.getRenderingConfig()}) - // If the type is “unknown” or “markdown”, we assume the markdown + // If the type is “” (unknown) or “markdown”, we assume the markdown // generation has been performed. Given the input: `a line`, markdown // specifies the HTML `<p>a line</p>\n`. When dealing with documents as a // whole, this is OK. When dealing with an `{{ .Inner }}` block in Hugo, @@ -442,12 +367,9 @@ func renderShortcode( // 1. Check to see if inner has a newline in it. If so, the Inner data is // unchanged. // 2 If inner does not have a newline, strip the wrapping <p> block and - // the newline. This was previously tricked out by wrapping shortcode - // substitutions in <div>HUGOSHORTCODE-1</div> which prevents the - // generation, but means that you can’t use shortcodes inside of - // markdown structures itself (e.g., `[foo]({{% ref foo.md %}})`). - switch p.Markup { - case "unknown", "markdown": + // the newline. + switch p.m.markup { + case "", "markdown": if match, _ := regexp.MatchString(innerNewlineRegexp, inner); !match { cleaner, err := regexp.Compile(innerCleanupRegexp) @@ -465,147 +387,71 @@ func renderShortcode( } - s, err := renderShortcodeWithPage(tmpl, data) + result, err := renderShortcodeWithPage(tmpl, data) if err != nil && sc.isInline { fe := herrors.ToFileError("html", err) l1, l2 := p.posFromPage(sc.pos).LineNumber, fe.Position().LineNumber fe = herrors.ToFileErrorWithLineNumber(fe, l1+l2-1) - return "", fe - } - - return s, err -} - -// The delta represents new output format-versions of the shortcodes, -// which, combined with the ones that do not have alternative representations, -// builds a complete set ready for a full rebuild of the Page content. -// This method returns false if there are no new shortcode variants in the -// current rendering context's output format. This mean we can safely reuse -// the content from the previous output format, if any. -func (s *shortcodeHandler) updateDelta() bool { - s.init.Do(func() { - s.contentShortcodes = s.createShortcodeRenderers(s.p.withoutContent()) - }) - - if !s.p.shouldRenderTo(s.p.s.rc.Format) { - // TODO(bep) add test for this re translations - return false + return "", false, fe } - of := s.p.s.rc.Format - contentShortcodes := s.contentShortcodesForOutputFormat(of) - if s.contentShortcodesDelta == nil || s.contentShortcodesDelta.Len() == 0 { - s.contentShortcodesDelta = contentShortcodes - return true - } - - delta := newOrderedMap() - - for _, k := range contentShortcodes.Keys() { - if !s.contentShortcodesDelta.Contains(k) { - v, _ := contentShortcodes.Get(k) - delta.Add(k, v) - } - } - - s.contentShortcodesDelta = delta - - return delta.Len() > 0 + return result, hasVariants, err } -func (s *shortcodeHandler) clearDelta() { - if s == nil { - return - } - s.contentShortcodesDelta = newOrderedMap() +func (s *shortcodeHandler) hasShortcodes() bool { + return len(s.shortcodes) > 0 } -func (s *shortcodeHandler) contentShortcodesForOutputFormat(f output.Format) *orderedMap { - contentShortcodesForOuputFormat := newOrderedMap() - lang := s.p.Lang() - - for _, key := range s.shortcodes.Keys() { - shortcodePlaceholder := key.(string) +func (s *shortcodeHandler) renderShortcodesForPage(p *pageState, f output.Format) (map[string]string, bool, error) { - key := newScKeyFromLangAndOutputFormat(lang, f, shortcodePlaceholder) - renderFn, found := s.contentShortcodes.Get(key) - - if !found { - key.OutputFormat = "" - renderFn, found = s.contentShortcodes.Get(key) - } - - // Fall back to HTML - if !found && key.Suffix != "html" { - key.Suffix = "html" - renderFn, found = s.contentShortcodes.Get(key) - if !found { - key.OutputFormat = "HTML" - renderFn, found = s.contentShortcodes.Get(key) - } - } + rendered := make(map[string]string) - if !found { - panic(fmt.Sprintf("Shortcode %q could not be found", shortcodePlaceholder)) - } - contentShortcodesForOuputFormat.Add(newScKeyFromLangAndOutputFormat(lang, f, shortcodePlaceholder), renderFn) + tplVariants := tpl.TemplateVariants{ + Language: p.Language().Lang, + OutputFormat: f, } - return contentShortcodesForOuputFormat -} - -func (s *shortcodeHandler) executeShortcodesForDelta(p *PageWithoutContent) error { + var hasVariants bool - for _, k := range s.contentShortcodesDelta.Keys() { - render := s.contentShortcodesDelta.getShortcodeRenderer(k) - renderedShortcode, err := render() + for _, v := range s.shortcodes { + s, more, err := renderShortcode(0, s.s, tplVariants, v, nil, p) if err != nil { - sc := s.shortcodes.getShortcode(k.(scKey).ShortcodePlaceholder) - if sc != nil { - err = p.errWithFileContext(p.parseError(_errors.Wrapf(err, "failed to render shortcode %q", sc.name), p.source.parsed.Input(), sc.pos)) - } - - p.s.SendError(err) - continue + err = p.parseError(_errors.Wrapf(err, "failed to render shortcode %q", v.name), p.source.parsed.Input(), v.pos) + return nil, false, err } + hasVariants = hasVariants || more + rendered[v.placeholder] = s - s.renderedShortcodes[k.(scKey).ShortcodePlaceholder] = renderedShortcode } - return nil - + return rendered, hasVariants, nil } -func (s *shortcodeHandler) createShortcodeRenderers(p *PageWithoutContent) *orderedMap { - - shortcodeRenderers := newOrderedMap() +var errShortCodeIllegalState = errors.New("Illegal shortcode state") - for _, k := range s.shortcodes.Keys() { - v := s.shortcodes.getShortcode(k) - prepared := s.prepareShortcodeForPage(k.(string), v, nil, p) - for kk, vv := range prepared { - shortcodeRenderers.Add(kk, vv) - } +func (s *shortcodeHandler) parseError(err error, input []byte, pos int) error { + if s.p != nil { + return s.p.parseError(err, input, pos) } - - return shortcodeRenderers + return err } -var errShortCodeIllegalState = errors.New("Illegal shortcode state") - // pageTokens state: // - before: positioned just before the shortcode start // - after: shortcode(s) consumed (plural when they are nested) -func (s *shortcodeHandler) extractShortcode(ordinal int, pt *pageparser.Iterator, p *Page) (*shortcode, error) { +func (s *shortcodeHandler) extractShortcode(ordinal, level int, pt *pageparser.Iterator) (*shortcode, error) { + if s == nil { + panic("handler nil") + } sc := &shortcode{ordinal: ordinal} - var isInner = false var cnt = 0 var nestedOrdinal = 0 + var nextLevel = level + 1 fail := func(err error, i pageparser.Item) error { - return p.parseError(err, pt.Input(), i.Pos) + return s.parseError(err, pt.Input(), i.Pos) } Loop: @@ -613,9 +459,6 @@ Loop: currItem := pt.Next() switch { case currItem.IsLeftShortcodeDelim(): - if sc.pos == 0 { - sc.pos = currItem.Pos - } next := pt.Peek() if next.IsShortcodeClose() { continue @@ -624,7 +467,7 @@ Loop: if cnt > 0 { // nested shortcode; append it to inner content pt.Backup() - nested, err := s.extractShortcode(nestedOrdinal, pt, p) + nested, err := s.extractShortcode(nestedOrdinal, nextLevel, pt) nestedOrdinal++ if nested.name != "" { s.nameSet[nested.name] = true @@ -644,13 +487,13 @@ Loop: case currItem.IsRightShortcodeDelim(): // we trust the template on this: // if there's no inner, we're done - if !sc.isInline && !isInner { + if !sc.isInline && !sc.info.IsInner { return sc, nil } case currItem.IsShortcodeClose(): next := pt.Peek() - if !sc.isInline && !isInner { + if !sc.isInline && !sc.info.IsInner { if next.IsError() { // return that error, more specific continue @@ -670,24 +513,21 @@ Loop: case currItem.IsText(): sc.inner = append(sc.inner, currItem.ValStr()) case currItem.IsShortcodeName(): + sc.name = currItem.ValStr() + + // Check if the template expects inner content. // We pick the first template for an arbitrary output format // if more than one. It is "all inner or no inner". - tmpl := getShortcodeTemplateForTemplateKey(scKey{}, sc.name, p.s.Tmpl) - if tmpl == nil { - return sc, fail(_errors.Errorf("template for shortcode %q not found", sc.name), currItem) - } - - var err error - isInner, err = isInnerShortcode(tmpl.(tpl.TemplateExecutor)) - if err != nil { - return sc, fail(_errors.Wrapf(err, "failed to handle template for shortcode %q", sc.name), currItem) + tmpl, found, _ := s.s.Tmpl.LookupVariant(sc.name, tpl.TemplateVariants{}) + if !found { + return nil, _errors.Errorf("template for shortcode %q not found", sc.name) } + sc.info = tmpl.(tpl.TemplateInfoProvider).TemplateInfo() case currItem.IsInlineShortcodeName(): sc.name = currItem.ValStr() sc.isInline = true - case currItem.IsShortcodeParam(): if !pt.IsValueNext() { continue @@ -721,7 +561,6 @@ Loop: } } - case currItem.IsDone(): // handled by caller pt.Backup() @@ -732,11 +571,9 @@ Loop: return sc, nil } -var shortCodeStart = []byte("{{") - -// Replace prefixed shortcode tokens (HUGOSHORTCODE-1, HUGOSHORTCODE-2) with the real content. +// Replace prefixed shortcode tokens with the real content. // Note: This function will rewrite the input slice. -func replaceShortcodeTokens(source []byte, prefix string, replacements map[string]string) ([]byte, error) { +func replaceShortcodeTokens(source []byte, replacements map[string]string) ([]byte, error) { if len(replacements) == 0 { return source, nil @@ -744,7 +581,7 @@ func replaceShortcodeTokens(source []byte, prefix string, replacements map[strin start := 0 - pre := []byte("HAHA" + prefix) + pre := []byte(shortcodePlaceholderPrefix) post := []byte("HBHB") pStart := []byte("<p>") pEnd := []byte("</p>") @@ -781,54 +618,11 @@ func replaceShortcodeTokens(source []byte, prefix string, replacements map[strin return source, nil } -func getShortcodeTemplateForTemplateKey(key scKey, shortcodeName string, t tpl.TemplateFinder) tpl.Template { - isInnerShortcodeCache.RLock() - defer isInnerShortcodeCache.RUnlock() - - var names []string - - suffix := strings.ToLower(key.Suffix) - outFormat := strings.ToLower(key.OutputFormat) - lang := strings.ToLower(key.Lang) - - if outFormat != "" && suffix != "" { - if lang != "" { - names = append(names, fmt.Sprintf("%s.%s.%s.%s", shortcodeName, lang, outFormat, suffix)) - } - names = append(names, fmt.Sprintf("%s.%s.%s", shortcodeName, outFormat, suffix)) - } - - if suffix != "" { - if lang != "" { - names = append(names, fmt.Sprintf("%s.%s.%s", shortcodeName, lang, suffix)) - } - names = append(names, fmt.Sprintf("%s.%s", shortcodeName, suffix)) - } - - names = append(names, shortcodeName) - - for _, name := range names { - - if x, found := t.Lookup("shortcodes/" + name); found { - return x - } - if x, found := t.Lookup("theme/shortcodes/" + name); found { - return x - } - if x, found := t.Lookup("_internal/shortcodes/" + name); found { - return x - } - } - return nil -} - func renderShortcodeWithPage(tmpl tpl.Template, data *ShortcodeWithPage) (string, error) { buffer := bp.GetBuffer() defer bp.PutBuffer(buffer) - isInnerShortcodeCache.RLock() err := tmpl.Execute(buffer, data) - isInnerShortcodeCache.RUnlock() if err != nil { return "", _errors.Wrap(err, "failed to process shortcode") } diff --git a/hugolib/shortcode_page.go b/hugolib/shortcode_page.go new file mode 100644 index 000000000..e8a3a37e1 --- /dev/null +++ b/hugolib/shortcode_page.go @@ -0,0 +1,56 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "html/template" + + "github.com/gohugoio/hugo/resources/page" +) + +var tocShortcodePlaceholder = createShortcodePlaceholder("TOC", 0) + +// This is sent to the shortcodes. They cannot access the content +// they're a part of. It would cause an infinite regress. +// +// Go doesn't support virtual methods, so this careful dance is currently (I think) +// the best we can do. +type pageForShortcode struct { + page.PageWithoutContent + page.ContentProvider + + // We need to replace it after we have rendered it, so provide a + // temporary placeholder. + toc template.HTML + + p *pageState +} + +func newPageForShortcode(p *pageState) page.Page { + return &pageForShortcode{ + PageWithoutContent: p, + ContentProvider: page.NopPage, + toc: template.HTML(tocShortcodePlaceholder), + p: p, + } +} + +func (p *pageForShortcode) page() page.Page { + return p.PageWithoutContent.(page.Page) +} + +func (p *pageForShortcode) TableOfContents() template.HTML { + p.p.enablePlaceholders() + return p.toc +} diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index 16ff0b780..f1603eeeb 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,9 +16,13 @@ package hugolib import ( "fmt" "path/filepath" - "reflect" "regexp" - "sort" + + "reflect" + + "github.com/gohugoio/hugo/parser/pageparser" + "github.com/gohugoio/hugo/resources/page" + "strings" "testing" @@ -26,34 +30,14 @@ import ( "github.com/spf13/afero" - "github.com/gohugoio/hugo/output" - - "github.com/gohugoio/hugo/media" - "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/tpl" + "github.com/spf13/cast" "github.com/stretchr/testify/require" ) -// TODO(bep) remove -func pageFromString(in, filename string, shortcodePlaceholderFn func() string, withTemplate ...func(templ tpl.TemplateHandler) error) (*Page, error) { - var err error - cfg, fs := newTestCfg() - - d := deps.DepsCfg{Cfg: cfg, Fs: fs, WithTemplate: withTemplate[0]} - - s, err := NewSiteForCfg(d) - if err != nil { - return nil, err - } - - s.shortcodePlaceholderFunc = shortcodePlaceholderFn - - return s.newPageFrom(strings.NewReader(in), filename) -} - func CheckShortCodeMatch(t *testing.T, input, expected string, withTemplate func(templ tpl.TemplateHandler) error) { CheckShortCodeMatchAndError(t, input, expected, withTemplate, false) } @@ -85,16 +69,16 @@ title: "Title" t.Fatalf("No error from shortcode") } - require.Len(t, h.Sites[0].RegularPages, 1) + require.Len(t, h.Sites[0].RegularPages(), 1) - output := strings.TrimSpace(string(h.Sites[0].RegularPages[0].content())) + output := strings.TrimSpace(content(h.Sites[0].RegularPages()[0])) output = strings.TrimPrefix(output, "<p>") output = strings.TrimSuffix(output, "</p>") expected = strings.TrimSpace(expected) if output != expected { - t.Fatalf("Shortcode render didn't match. got \n%q but expected \n%q", output, expected) + Fatalf(t, "Shortcode render didn't match. got \n%q but expected \n%q", output, expected) } } @@ -161,6 +145,28 @@ func TestShortcodeRelated(t *testing.T) { CheckShortCodeMatch(t, "{{< a >}}", "0", wt) } +func TestShortcodeInnerMarkup(t *testing.T) { + t.Parallel() + wt := func(tem tpl.TemplateHandler) error { + tem.AddTemplate("shortcodes/a.html", `<div>{{ .Inner }}</div>`) + tem.AddTemplate("shortcodes/b.html", `**Bold**: <div>{{ .Inner }}</div>`) + return nil + } + + CheckShortCodeMatch(t, + "{{< a >}}B: <div>{{% b %}}**Bold**{{% /b %}}</div>{{< /a >}}", + // This assertion looks odd, but is correct: for inner shortcodes with + // the {{% we treats the .Inner content as markup, but not the shortcode + // itself. + "<div>B: <div>**Bold**: <div><strong>Bold</strong></div></div></div>", + wt) + + CheckShortCodeMatch(t, + "{{% b %}}This is **B**: {{< b >}}This is B{{< /b>}}{{% /b %}}", + "<strong>Bold</strong>: <div>This is <strong>B</strong>: <strong>Bold</strong>: <div>This is B</div></div>", + wt) +} + // some repro issues for panics in Go Fuzz testing func TestNamedParamSC(t *testing.T) { @@ -188,7 +194,7 @@ func TestNestedNamedMissingParam(t *testing.T) { } CheckShortCodeMatch(t, `{{% acc %}}{{% div %}}d1{{% /div %}}{{% div2 %}}d2{{% /div2 %}}{{% /acc %}}`, - "<div class=\"acc\"><div >d1</div><div >d2</div>\n</div>", wt) + "<div class=\"acc\"><div >d1</div><div >d2</div></div>", wt) } func TestIsNamedParamsSC(t *testing.T) { @@ -218,39 +224,18 @@ func TestInnerSC(t *testing.T) { func TestInnerSCWithMarkdown(t *testing.T) { t.Parallel() wt := func(tem tpl.TemplateHandler) error { - tem.AddTemplate("_internal/shortcodes/inside.html", `<div{{with .Get "class"}} class="{{.}}"{{end}}>{{ .Inner }}</div>`) - return nil - } - CheckShortCodeMatch(t, `{{% inside %}} -# More Here - -[link](http://spf13.com) and text - -{{% /inside %}}`, "<div><h1 id=\"more-here\">More Here</h1>\n\n<p><a href=\"http://spf13.com\">link</a> and text</p>\n</div>", wt) -} - -func TestInnerSCWithAndWithoutMarkdown(t *testing.T) { - t.Parallel() - wt := func(tem tpl.TemplateHandler) error { - tem.AddTemplate("_internal/shortcodes/inside.html", `<div{{with .Get "class"}} class="{{.}}"{{end}}>{{ .Inner }}</div>`) + // Note: In Hugo 0.55 we made it so any outer {{%'s inner content was rendered as part of the surrounding + // markup. This solved lots of problems, but it also meant that this test had to be adjusted. + tem.AddTemplate("_internal/shortcodes/wrapper.html", `<div{{with .Get "class"}} class="{{.}}"{{end}}>{{ .Inner }}</div>`) + tem.AddTemplate("_internal/shortcodes/inside.html", `{{ .Inner }}`) return nil } - CheckShortCodeMatch(t, `{{% inside %}} + CheckShortCodeMatch(t, `{{< wrapper >}}{{% inside %}} # More Here [link](http://spf13.com) and text -{{% /inside %}} - -And then: - -{{< inside >}} -# More Here - -This is **plain** text. - -{{< /inside >}} -`, "<div><h1 id=\"more-here\">More Here</h1>\n\n<p><a href=\"http://spf13.com\">link</a> and text</p>\n</div>\n\n<p>And then:</p>\n\n<div>\n# More Here\n\nThis is **plain** text.\n\n</div>", wt) +{{% /inside %}}{{< /wrapper >}}`, "<div><h1 id=\"more-here\">More Here</h1>\n\n<p><a href=\"http://spf13.com\">link</a> and text</p>\n</div>", wt) } func TestEmbeddedSC(t *testing.T) { @@ -266,7 +251,7 @@ func TestNestedSC(t *testing.T) { tem.AddTemplate("_internal/shortcodes/scn2.html", `<div>SC2</div>`) return nil } - CheckShortCodeMatch(t, `{{% scn1 %}}{{% scn2 %}}{{% /scn1 %}}`, "<div>Outer, inner is <div>SC2</div>\n</div>", wt) + CheckShortCodeMatch(t, `{{% scn1 %}}{{% scn2 %}}{{% /scn1 %}}`, "<div>Outer, inner is <div>SC2</div></div>", wt) CheckShortCodeMatch(t, `{{< scn1 >}}{{% scn2 %}}{{< /scn1 >}}`, "<div>Outer, inner is <div>SC2</div></div>", wt) } @@ -355,136 +340,100 @@ func TestShortcodeWrappedInPIssue(t *testing.T) { `, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", wt) } -const testScPlaceholderRegexp = "HAHAHUGOSHORTCODE-\\d+HBHB" - func TestExtractShortcodes(t *testing.T) { t.Parallel() + b := newTestSitesBuilder(t).WithSimpleConfigFile() + + b.WithTemplates( + "default/single.html", `EMPTY`, + "_internal/shortcodes/tag.html", `tag`, + "_internal/shortcodes/legacytag.html", `{{ $_hugo_config := "{ \"version\": 1 }" }}tag`, + "_internal/shortcodes/sc1.html", `sc1`, + "_internal/shortcodes/sc2.html", `sc2`, + "_internal/shortcodes/inner.html", `{{with .Inner }}{{ . }}{{ end }}`, + "_internal/shortcodes/inner2.html", `{{.Inner}}`, + "_internal/shortcodes/inner3.html", `{{.Inner}}`, + ).WithContent("page.md", `--- +title: "Shortcodes Galore!" +--- +`) - for i, this := range []struct { - name string - input string - expectShortCodes string - expect interface{} - expectErrorMsg string - }{ - {"text", "Some text.", "map[]", "Some text.", ""}, - {"invalid right delim", "{{< tag }}", "", false, "unrecognized character"}, - {"invalid close", "\n{{< /tag >}}", "", false, "got closing shortcode, but none is open"}, - {"invalid close2", "\n\n{{< tag >}}{{< /anotherTag >}}", "", false, "closing tag for shortcode 'anotherTag' does not match start tag"}, - {"unterminated quote 1", `{{< figure src="im caption="S" >}}`, "", false, "got pos"}, - {"unterminated quote 1", `{{< figure src="im" caption="S >}}`, "", false, "unterm"}, - {"one shortcode, no markup", "{{< tag >}}", "", testScPlaceholderRegexp, ""}, - {"one shortcode, markup", "{{% tag %}}", "", testScPlaceholderRegexp, ""}, - {"one pos param", "{{% tag param1 %}}", `tag([\"param1\"], true){[]}"]`, testScPlaceholderRegexp, ""}, - {"two pos params", "{{< tag param1 param2>}}", `tag([\"param1\" \"param2\"], false){[]}"]`, testScPlaceholderRegexp, ""}, - {"one named param", `{{% tag param1="value" %}}`, `tag([\"param1:value\"], true){[]}`, testScPlaceholderRegexp, ""}, - {"two named params", `{{< tag param1="value1" param2="value2" >}}`, `tag([\"param1:value1\" \"param2:value2\"], false){[]}"]`, - testScPlaceholderRegexp, ""}, - {"inner", `Some text. {{< inner >}}Inner Content{{< / inner >}}. Some more text.`, `inner([], false){[Inner Content]}`, - fmt.Sprintf("Some text. %s. Some more text.", testScPlaceholderRegexp), ""}, - // issue #934 - {"inner self-closing", `Some text. {{< inner />}}. Some more text.`, `inner([], false){[]}`, - fmt.Sprintf("Some text. %s. Some more text.", testScPlaceholderRegexp), ""}, - {"close, but not inner", "{{< tag >}}foo{{< /tag >}}", "", false, `shortcode "tag" has no .Inner, yet a closing tag was provided`}, - {"nested inner", `Inner->{{< inner >}}Inner Content->{{% inner2 param1 %}}inner2txt{{% /inner2 %}}Inner close->{{< / inner >}}<-done`, - `inner([], false){[Inner Content-> inner2([\"param1\"], true){[inner2txt]} Inner close->]}`, - fmt.Sprintf("Inner->%s<-done", testScPlaceholderRegexp), ""}, - {"nested, nested inner", `Inner->{{< inner >}}inner2->{{% inner2 param1 %}}inner2txt->inner3{{< inner3>}}inner3txt{{</ inner3 >}}{{% /inner2 %}}final close->{{< / inner >}}<-done`, - `inner([], false){[inner2-> inner2([\"param1\"], true){[inner2txt->inner3 inner3(%!q(<nil>), false){[inner3txt]}]} final close->`, - fmt.Sprintf("Inner->%s<-done", testScPlaceholderRegexp), ""}, - {"two inner", `Some text. {{% inner %}}First **Inner** Content{{% / inner %}} {{< inner >}}Inner **Content**{{< / inner >}}. Some more text.`, - `map["HAHAHUGOSHORTCODE-1HBHB:inner([], true){[First **Inner** Content]}" "HAHAHUGOSHORTCODE-2HBHB:inner([], false){[Inner **Content**]}"]`, - fmt.Sprintf("Some text. %s %s. Some more text.", testScPlaceholderRegexp, testScPlaceholderRegexp), ""}, - {"closed without content", `Some text. {{< inner param1 >}}{{< / inner >}}. Some more text.`, `inner([\"param1\"], false){[]}`, - fmt.Sprintf("Some text. %s. Some more text.", testScPlaceholderRegexp), ""}, - {"two shortcodes", "{{< sc1 >}}{{< sc2 >}}", - `map["HAHAHUGOSHORTCODE-1HBHB:sc1([], false){[]}" "HAHAHUGOSHORTCODE-2HBHB:sc2([], false){[]}"]`, - testScPlaceholderRegexp + testScPlaceholderRegexp, ""}, - {"mix of shortcodes", `Hello {{< sc1 >}}world{{% sc2 p2="2"%}}. And that's it.`, - `map["HAHAHUGOSHORTCODE-1HBHB:sc1([], false){[]}" "HAHAHUGOSHORTCODE-2HBHB:sc2([\"p2:2\"]`, - fmt.Sprintf("Hello %sworld%s. And that's it.", testScPlaceholderRegexp, testScPlaceholderRegexp), ""}, - {"mix with inner", `Hello {{< sc1 >}}world{{% inner p2="2"%}}Inner{{%/ inner %}}. And that's it.`, - `map["HAHAHUGOSHORTCODE-1HBHB:sc1([], false){[]}" "HAHAHUGOSHORTCODE-2HBHB:inner([\"p2:2\"], true){[Inner]}"]`, - fmt.Sprintf("Hello %sworld%s. And that's it.", testScPlaceholderRegexp, testScPlaceholderRegexp), ""}, - } { + b.CreateSites().Build(BuildCfg{}) - pageInput := simplePage + this.input + s := b.H.Sites[0] - counter := 0 - placeholderFunc := func() string { - counter++ - return fmt.Sprintf("HAHA%s-%dHBHB", shortcodePlaceholderPrefix, counter) + /*errCheck := func(s string) func(name string, assert *require.Assertions, shortcode *shortcode, err error) { + return func(name string, assert *require.Assertions, shortcode *shortcode, err error) { + assert.Error(err, name) + assert.Equal(s, err.Error(), name) } + }*/ - p, err := pageFromString(pageInput, "simple.md", placeholderFunc, func(templ tpl.TemplateHandler) error { - templ.AddTemplate("_internal/shortcodes/tag.html", `tag`) - templ.AddTemplate("_internal/shortcodes/sc1.html", `sc1`) - templ.AddTemplate("_internal/shortcodes/sc2.html", `sc2`) - templ.AddTemplate("_internal/shortcodes/inner.html", `{{with .Inner }}{{ . }}{{ end }}`) - templ.AddTemplate("_internal/shortcodes/inner2.html", `{{.Inner}}`) - templ.AddTemplate("_internal/shortcodes/inner3.html", `{{.Inner}}`) - return nil - }) + // Make it more regexp friendly + strReplacer := strings.NewReplacer("[", "{", "]", "}") - if b, ok := this.expect.(bool); ok && !b { - if err == nil { - t.Fatalf("[%d] %s: ExtractShortcodes didn't return an expected error", i, this.name) - } else { - r := regexp.MustCompile(this.expectErrorMsg) - if !r.MatchString(err.Error()) { - t.Fatalf("[%d] %s: ExtractShortcodes didn't return an expected error message, got\n%s but expected\n%s", - i, this.name, err.Error(), this.expectErrorMsg) - } - } - continue - } else { - if err != nil { - t.Fatalf("[%d] %s: failed: %q", i, this.name, err) - } + str := func(s *shortcode) string { + if s == nil { + return "<nil>" } + return strReplacer.Replace(fmt.Sprintf("%s;inline:%t;closing:%t;inner:%v;params:%v;ordinal:%d;markup:%t;version:%d;pos:%d", + s.name, s.isInline, s.isClosing, s.inner, s.params, s.ordinal, s.doMarkup, s.info.Config.Version, s.pos)) + } - shortCodes := p.shortcodeState.shortcodes - contentReplaced := string(p.workContent) - - var expected string - av := reflect.ValueOf(this.expect) - switch av.Kind() { - case reflect.String: - expected = av.String() + regexpCheck := func(re string) func(assert *require.Assertions, shortcode *shortcode, err error) { + return func(assert *require.Assertions, shortcode *shortcode, err error) { + assert.NoError(err) + got := str(shortcode) + assert.Regexp(regexp.MustCompile(re), got, got) } + } - r, err := regexp.Compile(expected) + for _, test := range []struct { + name string + input string + check func(assert *require.Assertions, shortcode *shortcode, err error) + }{ + {"one shortcode, no markup", "{{< tag >}}", regexpCheck("tag.*closing:false.*markup:false")}, + {"one shortcode, markup", "{{% tag %}}", regexpCheck("tag.*closing:false.*markup:true;version:2")}, + {"one shortcode, markup, legacy", "{{% legacytag %}}", regexpCheck("tag.*closing:false.*markup:true;version:1")}, + {"outer shortcode markup", "{{% inner %}}{{< tag >}}{{% /inner %}}", regexpCheck("inner.*closing:true.*markup:true")}, + {"inner shortcode markup", "{{< inner >}}{{% tag %}}{{< /inner >}}", regexpCheck("inner.*closing:true.*;markup:false;version:2")}, + {"one pos param", "{{% tag param1 %}}", regexpCheck("tag.*params:{param1}")}, + {"two pos params", "{{< tag param1 param2>}}", regexpCheck("tag.*params:{param1 param2}")}, + {"one named param", `{{% tag param1="value" %}}`, regexpCheck("tag.*params:map{param1:value}")}, + {"two named params", `{{< tag param1="value1" param2="value2" >}}`, regexpCheck("tag.*params:map{param\\d:value\\d param\\d:value\\d}")}, + {"inner", `{{< inner >}}Inner Content{{< / inner >}}`, regexpCheck("inner;inline:false;closing:true;inner:{Inner Content};")}, + // issue #934 + {"inner self-closing", `{{< inner />}}`, regexpCheck("inner;.*inner:{}")}, + {"nested inner", `{{< inner >}}Inner Content->{{% inner2 param1 %}}inner2txt{{% /inner2 %}}Inner close->{{< / inner >}}`, + regexpCheck("inner;.*inner:{Inner Content->.*Inner close->}")}, + {"nested, nested inner", `{{< inner >}}inner2->{{% inner2 param1 %}}inner2txt->inner3{{< inner3>}}inner3txt{{</ inner3 >}}{{% /inner2 %}}final close->{{< / inner >}}`, + regexpCheck("inner:{inner2-> inner2.*{{inner2txt->inner3.*final close->}")}, + {"closed without content", `{{< inner param1 >}}{{< / inner >}}`, regexpCheck("inner.*inner:{}")}, + {"inline", `{{< my.inline >}}Hi{{< /my.inline >}}`, regexpCheck("my.inline;inline:true;closing:true;inner:{Hi};")}, + } { - if err != nil { - t.Fatalf("[%d] %s: Failed to compile regexp %q: %q", i, this.name, expected, err) - } + t.Run(test.name, func(t *testing.T) { + assert := require.New(t) - if strings.Count(contentReplaced, shortcodePlaceholderPrefix) != shortCodes.Len() { - t.Fatalf("[%d] %s: Not enough placeholders, found %d", i, this.name, shortCodes.Len()) - } + counter := 0 + placeholderFunc := func() string { + counter++ + return fmt.Sprintf("HAHA%s-%dHBHB", shortcodePlaceholderPrefix, counter) + } - if !r.MatchString(contentReplaced) { - t.Fatalf("[%d] %s: Shortcode extract didn't match. got %q but expected %q", i, this.name, contentReplaced, expected) - } + p, err := pageparser.ParseMain(strings.NewReader(test.input), pageparser.Config{}) + assert.NoError(err) + handler := newShortcodeHandler(nil, s, placeholderFunc) + iter := p.Iterator() - for _, placeHolder := range shortCodes.Keys() { - sc := shortCodes.getShortcode(placeHolder) - if !strings.Contains(contentReplaced, placeHolder.(string)) { - t.Fatalf("[%d] %s: Output does not contain placeholder %q", i, this.name, placeHolder) - } + short, err := handler.extractShortcode(0, 0, iter) - if sc.params == nil { - t.Fatalf("[%d] %s: Params is nil for shortcode '%s'", i, this.name, sc.name) - } - } + test.check(assert, short, err) - if this.expectShortCodes != "" { - shortCodesAsStr := fmt.Sprintf("map%q", collectAndSortShortcodes(shortCodes)) - if !strings.Contains(shortCodesAsStr, this.expectShortCodes) { - t.Fatalf("[%d] %s: Shortcodes not as expected, got\n%s but expected\n%s", i, this.name, shortCodesAsStr, this.expectShortCodes) - } - } + }) } + } func TestShortcodesInSite(t *testing.T) { @@ -495,7 +444,7 @@ func TestShortcodesInSite(t *testing.T) { contentPath string content string outFile string - expected string + expected interface{} }{ {"sect/doc1.md", `a{{< b >}}c`, filepath.FromSlash("public/sect/doc1/index.html"), "<p>abc</p>\n"}, @@ -542,7 +491,7 @@ e`, // #2192 #2209: Shortcodes in markdown headers {"sect/doc5.md", `# {{< b >}} ## {{% c %}}`, - filepath.FromSlash("public/sect/doc5/index.html"), "\n\n<h1 id=\"hahahugoshortcode-1hbhb\">b</h1>\n\n<h2 id=\"hahahugoshortcode-2hbhb\">c</h2>\n"}, + filepath.FromSlash("public/sect/doc5/index.html"), `-hbhb">b</h1>`}, // #2223 pygments {"sect/doc6.md", "\n```bash\nb = {{< b >}} c = {{% c %}}\n```\n", filepath.FromSlash("public/sect/doc6/index.html"), @@ -591,7 +540,7 @@ tags: } addTemplates := func(templ tpl.TemplateHandler) error { - templ.AddTemplate("_default/single.html", "{{.Content}}") + templ.AddTemplate("_default/single.html", "{{.Content}} Word Count: {{ .WordCount }}") templ.AddTemplate("_internal/shortcodes/b.html", `b`) templ.AddTemplate("_internal/shortcodes/c.html", `c`) @@ -616,21 +565,21 @@ tags: writeSourcesToSource(t, "content", fs, sources...) s := buildSingleSite(t, deps.DepsCfg{WithTemplate: addTemplates, Fs: fs, Cfg: cfg}, BuildCfg{}) - th := testHelper{s.Cfg, s.Fs, t} - - for _, test := range tests { - if strings.HasSuffix(test.contentPath, ".ad") && !helpers.HasAsciidoc() { - fmt.Println("Skip Asciidoc test case as no Asciidoc present.") - continue - } else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() { - fmt.Println("Skip Rst test case as no rst2html present.") - continue - } else if strings.Contains(test.expected, "code") { - fmt.Println("Skip Pygments test case as no pygments present.") - continue - } - th.assertFileContent(test.outFile, test.expected) + for i, test := range tests { + t.Run(fmt.Sprintf("test=%d;contentPath=%s", i, test.contentPath), func(t *testing.T) { + if strings.HasSuffix(test.contentPath, ".ad") && !helpers.HasAsciidoc() { + t.Skip("Skip Asciidoc test case as no Asciidoc present.") + } else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() { + t.Skip("Skip Rst test case as no rst2html present.") + } + + th := testHelper{s.Cfg, s.Fs, t} + + expected := cast.ToStringSlice(test.expected) + th.assertFileContent(test.outFile, expected...) + }) + } } @@ -703,9 +652,9 @@ CSV: {{< myShort >}} require.Len(t, h.Sites, 1) s := h.Sites[0] - home := s.getPage(KindHome) + home := s.getPage(page.KindHome) require.NotNil(t, home) - require.Len(t, home.outputFormats, 3) + require.Len(t, home.OutputFormats(), 3) th.assertFileContent("public/index.html", "Home HTML", @@ -763,19 +712,6 @@ CSV: {{< myShort >}} } -func collectAndSortShortcodes(shortcodes *orderedMap) []string { - var asArray []string - - for _, key := range shortcodes.Keys() { - sc := shortcodes.getShortcode(key) - asArray = append(asArray, fmt.Sprintf("%s:%s", key, sc)) - } - - sort.Strings(asArray) - return asArray - -} - func BenchmarkReplaceShortcodeTokens(b *testing.B) { type input struct { @@ -811,7 +747,7 @@ func BenchmarkReplaceShortcodeTokens(b *testing.B) { for j := range data { currIn := in[cnt] cnt++ - results, err := replaceShortcodeTokens(currIn.in, "HUGOSHORTCODE", currIn.replacements) + results, err := replaceShortcodeTokens(currIn.in, currIn.replacements) if err != nil { b.Fatalf("[%d] failed: %s", i, err) @@ -834,36 +770,36 @@ func TestReplaceShortcodeTokens(t *testing.T) { replacements map[string]string expect interface{} }{ - {"Hello HAHAPREFIX-1HBHB.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "World"}, "Hello World."}, - {"Hello HAHAPREFIX-1@}@.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "World"}, false}, - {"HAHAPREFIX2-1HBHB", "PREFIX2", map[string]string{"HAHAPREFIX2-1HBHB": "World"}, "World"}, + {"Hello HAHAHUGOSHORTCODE-1HBHB.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "World"}, "Hello World."}, + {"Hello HAHAHUGOSHORTCODE-1@}@.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "World"}, false}, + {"HAHAHUGOSHORTCODE2-1HBHB", "PREFIX2", map[string]string{"HAHAHUGOSHORTCODE2-1HBHB": "World"}, "World"}, {"Hello World!", "PREFIX2", map[string]string{}, "Hello World!"}, - {"!HAHAPREFIX-1HBHB", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "World"}, "!World"}, - {"HAHAPREFIX-1HBHB!", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "World"}, "World!"}, - {"!HAHAPREFIX-1HBHB!", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "World"}, "!World!"}, - {"_{_PREFIX-1HBHB", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "World"}, "_{_PREFIX-1HBHB"}, - {"Hello HAHAPREFIX-1HBHB.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "To You My Old Friend Who Told Me This Fantastic Story"}, "Hello To You My Old Friend Who Told Me This Fantastic Story."}, - {"A HAHAA-1HBHB asdf HAHAA-2HBHB.", "A", map[string]string{"HAHAA-1HBHB": "v1", "HAHAA-2HBHB": "v2"}, "A v1 asdf v2."}, - {"Hello HAHAPREFIX2-1HBHB. Go HAHAPREFIX2-2HBHB, Go, Go HAHAPREFIX2-3HBHB Go Go!.", "PREFIX2", map[string]string{"HAHAPREFIX2-1HBHB": "Europe", "HAHAPREFIX2-2HBHB": "Jonny", "HAHAPREFIX2-3HBHB": "Johnny"}, "Hello Europe. Go Jonny, Go, Go Johnny Go Go!."}, - {"A HAHAPREFIX-2HBHB HAHAPREFIX-1HBHB.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "A", "HAHAPREFIX-2HBHB": "B"}, "A B A."}, - {"A HAHAPREFIX-1HBHB HAHAPREFIX-2", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "A"}, false}, - {"A HAHAPREFIX-1HBHB but not the second.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "A", "HAHAPREFIX-2HBHB": "B"}, "A A but not the second."}, - {"An HAHAPREFIX-1HBHB.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "A", "HAHAPREFIX-2HBHB": "B"}, "An A."}, - {"An HAHAPREFIX-1HBHB HAHAPREFIX-2HBHB.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "A", "HAHAPREFIX-2HBHB": "B"}, "An A B."}, - {"A HAHAPREFIX-1HBHB HAHAPREFIX-2HBHB HAHAPREFIX-3HBHB HAHAPREFIX-1HBHB HAHAPREFIX-3HBHB.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "A", "HAHAPREFIX-2HBHB": "B", "HAHAPREFIX-3HBHB": "C"}, "A A B C A C."}, - {"A HAHAPREFIX-1HBHB HAHAPREFIX-2HBHB HAHAPREFIX-3HBHB HAHAPREFIX-1HBHB HAHAPREFIX-3HBHB.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "A", "HAHAPREFIX-2HBHB": "B", "HAHAPREFIX-3HBHB": "C"}, "A A B C A C."}, + {"!HAHAHUGOSHORTCODE-1HBHB", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "World"}, "!World"}, + {"HAHAHUGOSHORTCODE-1HBHB!", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "World"}, "World!"}, + {"!HAHAHUGOSHORTCODE-1HBHB!", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "World"}, "!World!"}, + {"_{_PREFIX-1HBHB", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "World"}, "_{_PREFIX-1HBHB"}, + {"Hello HAHAHUGOSHORTCODE-1HBHB.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "To You My Old Friend Who Told Me This Fantastic Story"}, "Hello To You My Old Friend Who Told Me This Fantastic Story."}, + {"A HAHAHUGOSHORTCODE-1HBHB asdf HAHAHUGOSHORTCODE-2HBHB.", "A", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "v1", "HAHAHUGOSHORTCODE-2HBHB": "v2"}, "A v1 asdf v2."}, + {"Hello HAHAHUGOSHORTCODE2-1HBHB. Go HAHAHUGOSHORTCODE2-2HBHB, Go, Go HAHAHUGOSHORTCODE2-3HBHB Go Go!.", "PREFIX2", map[string]string{"HAHAHUGOSHORTCODE2-1HBHB": "Europe", "HAHAHUGOSHORTCODE2-2HBHB": "Jonny", "HAHAHUGOSHORTCODE2-3HBHB": "Johnny"}, "Hello Europe. Go Jonny, Go, Go Johnny Go Go!."}, + {"A HAHAHUGOSHORTCODE-2HBHB HAHAHUGOSHORTCODE-1HBHB.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "A", "HAHAHUGOSHORTCODE-2HBHB": "B"}, "A B A."}, + {"A HAHAHUGOSHORTCODE-1HBHB HAHAHUGOSHORTCODE-2", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "A"}, false}, + {"A HAHAHUGOSHORTCODE-1HBHB but not the second.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "A", "HAHAHUGOSHORTCODE-2HBHB": "B"}, "A A but not the second."}, + {"An HAHAHUGOSHORTCODE-1HBHB.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "A", "HAHAHUGOSHORTCODE-2HBHB": "B"}, "An A."}, + {"An HAHAHUGOSHORTCODE-1HBHB HAHAHUGOSHORTCODE-2HBHB.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "A", "HAHAHUGOSHORTCODE-2HBHB": "B"}, "An A B."}, + {"A HAHAHUGOSHORTCODE-1HBHB HAHAHUGOSHORTCODE-2HBHB HAHAHUGOSHORTCODE-3HBHB HAHAHUGOSHORTCODE-1HBHB HAHAHUGOSHORTCODE-3HBHB.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "A", "HAHAHUGOSHORTCODE-2HBHB": "B", "HAHAHUGOSHORTCODE-3HBHB": "C"}, "A A B C A C."}, + {"A HAHAHUGOSHORTCODE-1HBHB HAHAHUGOSHORTCODE-2HBHB HAHAHUGOSHORTCODE-3HBHB HAHAHUGOSHORTCODE-1HBHB HAHAHUGOSHORTCODE-3HBHB.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "A", "HAHAHUGOSHORTCODE-2HBHB": "B", "HAHAHUGOSHORTCODE-3HBHB": "C"}, "A A B C A C."}, // Issue #1148 remove p-tags 10 => - {"Hello <p>HAHAPREFIX-1HBHB</p>. END.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "World"}, "Hello World. END."}, - {"Hello <p>HAHAPREFIX-1HBHB</p>. <p>HAHAPREFIX-2HBHB</p> END.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "World", "HAHAPREFIX-2HBHB": "THE"}, "Hello World. THE END."}, - {"Hello <p>HAHAPREFIX-1HBHB. END</p>.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "World"}, "Hello <p>World. END</p>."}, - {"<p>Hello HAHAPREFIX-1HBHB</p>. END.", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "World"}, "<p>Hello World</p>. END."}, - {"Hello <p>HAHAPREFIX-1HBHB12", "PREFIX", map[string]string{"HAHAPREFIX-1HBHB": "World"}, "Hello <p>World12"}, - {"Hello HAHAP-1HBHB. HAHAP-1HBHB-HAHAP-1HBHB HAHAP-1HBHB HAHAP-1HBHB HAHAP-1HBHB END", "P", map[string]string{"HAHAP-1HBHB": strings.Repeat("BC", 100)}, + {"Hello <p>HAHAHUGOSHORTCODE-1HBHB</p>. END.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "World"}, "Hello World. END."}, + {"Hello <p>HAHAHUGOSHORTCODE-1HBHB</p>. <p>HAHAHUGOSHORTCODE-2HBHB</p> END.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "World", "HAHAHUGOSHORTCODE-2HBHB": "THE"}, "Hello World. THE END."}, + {"Hello <p>HAHAHUGOSHORTCODE-1HBHB. END</p>.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "World"}, "Hello <p>World. END</p>."}, + {"<p>Hello HAHAHUGOSHORTCODE-1HBHB</p>. END.", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "World"}, "<p>Hello World</p>. END."}, + {"Hello <p>HAHAHUGOSHORTCODE-1HBHB12", "PREFIX", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "World"}, "Hello <p>World12"}, + {"Hello HAHAHUGOSHORTCODE-1HBHB. HAHAHUGOSHORTCODE-1HBHB-HAHAHUGOSHORTCODE-1HBHB HAHAHUGOSHORTCODE-1HBHB HAHAHUGOSHORTCODE-1HBHB HAHAHUGOSHORTCODE-1HBHB END", "P", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": strings.Repeat("BC", 100)}, fmt.Sprintf("Hello %s. %s-%s %s %s %s END", strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100))}, } { - results, err := replaceShortcodeTokens([]byte(this.input), this.prefix, this.replacements) + results, err := replaceShortcodeTokens([]byte(this.input), this.replacements) if b, ok := this.expect.(bool); ok && !b { if err == nil { @@ -883,16 +819,6 @@ func TestReplaceShortcodeTokens(t *testing.T) { } -func TestScKey(t *testing.T) { - require.Equal(t, scKey{Suffix: "xml", ShortcodePlaceholder: "ABCD"}, - newScKey(media.XMLType, "ABCD")) - require.Equal(t, scKey{Lang: "en", Suffix: "html", OutputFormat: "AMP", ShortcodePlaceholder: "EFGH"}, - newScKeyFromLangAndOutputFormat("en", output.AMPFormat, "EFGH")) - require.Equal(t, scKey{Suffix: "html", ShortcodePlaceholder: "IJKL"}, - newDefaultScKey("IJKL")) - -} - func TestShortcodeGetContent(t *testing.T) { t.Parallel() assert := require.New(t) @@ -950,7 +876,7 @@ C-%s` builder.WithViper(v).WithContent(content...).WithTemplates(templates...).CreateSites().Build(BuildCfg{}) s := builder.H.Sites[0] - assert.Equal(3, len(s.RegularPages)) + assert.Equal(3, len(s.RegularPages())) builder.AssertFileContent("public/section1/index.html", "List Content: <p>Logo:P1:|P2:logo.png/PNG logo|:P1: P1:|P2:docs1p1/<p>C-s1p1</p>\n|", @@ -1017,7 +943,7 @@ weight: %d builder.WithContent(content...).WithTemplatesAdded(shortcodes...).CreateSites().Build(BuildCfg{}) s := builder.H.Sites[0] - assert.Equal(3, len(s.RegularPages)) + assert.Equal(3, len(s.RegularPages())) builder.AssertFileContent("public/en/p1/index.html", `v1: 0 sgo: |v2: 1 sgo: 0|v3: 2 sgo: 1|v4: 3 sgo: 2|v5: 4 sgo: 3`) builder.AssertFileContent("public/en/p1/index.html", `outer ordinal: 5 inner: @@ -1054,7 +980,7 @@ String: {{ . | safeHTML }} `).CreateSites().Build(BuildCfg{}) s := builder.H.Sites[0] - assert.Equal(1, len(s.RegularPages)) + assert.Equal(1, len(s.RegularPages())) builder.AssertFileContent("public/page/index.html", filepath.FromSlash("File: content/page.md"), diff --git a/hugolib/site.go b/hugolib/site.go index 43b398b70..be70db5ee 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,59 +22,54 @@ import ( "mime" "net/url" "os" + "path" "path/filepath" "sort" "strconv" "strings" "time" + "github.com/gohugoio/hugo/common/maps" + "github.com/pkg/errors" "github.com/gohugoio/hugo/common/text" - "github.com/gohugoio/hugo/hugofs" - - "github.com/gohugoio/hugo/common/herrors" - "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/publisher" _errors "github.com/pkg/errors" "github.com/gohugoio/hugo/langs" - src "github.com/gohugoio/hugo/source" - - "golang.org/x/sync/errgroup" + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/lazy" + "golang.org/x/sync/errgroup" "github.com/gohugoio/hugo/media" - "github.com/gohugoio/hugo/parser/metadecoders" - - "github.com/markbates/inflect" "github.com/fsnotify/fsnotify" bp "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugolib/pagemeta" + "github.com/gohugoio/hugo/navigation" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/related" "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/page/pagemeta" + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/tpl" + "github.com/spf13/afero" "github.com/spf13/cast" - "github.com/spf13/nitro" "github.com/spf13/viper" ) // used to indicate if run as a test. var testMode bool -var defaultTimer *nitro.B - // Site contains all the information relevant for constructing a static // site. The basic flow of information is as follows: // @@ -93,34 +88,27 @@ var defaultTimer *nitro.B // // 5. The entire collection of files is written to disk. type Site struct { - owner *HugoSites + + // The owning container. When multiple languages, there will be multiple + // sites. + h *HugoSites *PageCollections Taxonomies TaxonomyList - // Plural is what we get in the folder, so keep track of this mapping - // to get the singular form from that value. - taxonomiesPluralSingular map[string]string - - // This is temporary, see https://github.com/gohugoio/hugo/issues/2835 - // Maps "actors-gerard-depardieu" to "Gérard Depardieu" when preserveTaxonomyNames - // is set. - taxonomiesOrigKey map[string]string + taxonomyNodes taxonomyNodeInfos Sections Taxonomy Info SiteInfo - Menus Menus - timer *nitro.B layoutHandler *output.LayoutHandler - draftCount int - futureCount int - expiredCount int + buildStats *buildStats - Data map[string]interface{} - Language *langs.Language + language *langs.Language + + siteCfg siteConfigHolder disabledKinds map[string]bool @@ -137,7 +125,7 @@ type Site struct { outputFormatsConfig output.Formats mediaTypesConfig media.Types - siteConfig SiteConfig + siteConfigConfig SiteConfig // How to handle page front matter. frontmatterHandler pagemeta.FrontMatterHandler @@ -158,23 +146,162 @@ type Site struct { // The func used to title case titles. titleFunc func(s string) string - relatedDocsHandler *relatedDocsHandler + relatedDocsHandler *page.RelatedDocsHandler siteRefLinker - // Set in some tests - shortcodePlaceholderFunc func() string publisher publisher.Publisher + + menus navigation.Menus + + // Shortcut to the home page. Note that this may be nil if + // home page, for some odd reason, is disabled. + home *pageState + + // The last modification date of this site. + lastmod time.Time + + // Lazily loaded site dependencies + init *siteInit +} + +type siteConfigHolder struct { + sitemap config.Sitemap + taxonomiesConfig map[string]string + timeout time.Duration + hasCJKLanguage bool + enableEmoji bool +} + +// Lazily loaded site dependencies. +type siteInit struct { + prevNext *lazy.Init + prevNextInSection *lazy.Init + menus *lazy.Init +} + +func (init *siteInit) Reset() { + init.prevNext.Reset() + init.prevNextInSection.Reset() + init.menus.Reset() +} + +func (s *Site) initInit(init *lazy.Init, pctx pageContext) { + _, err := init.Do() + if err != nil { + s.h.FatalError(pctx.wrapError(err)) + } +} + +func (s *Site) prepareInits() { + s.init = &siteInit{} + + var init lazy.Init + + s.init.prevNext = init.Branch(func() (interface{}, error) { + regularPages := s.findWorkPagesByKind(page.KindPage) + for i, p := range regularPages { + if p.posNextPrev == nil { + continue + } + p.posNextPrev.nextPage = nil + p.posNextPrev.prevPage = nil + + if i > 0 { + p.posNextPrev.nextPage = regularPages[i-1] + } + + if i < len(regularPages)-1 { + p.posNextPrev.prevPage = regularPages[i+1] + } + } + return nil, nil + }) + + s.init.prevNextInSection = init.Branch(func() (interface{}, error) { + var rootSection []int + for i, p1 := range s.workAllPages { + if p1.IsPage() && p1.Section() == "" { + rootSection = append(rootSection, i) + } + if p1.IsSection() && len(p1.SectionsEntries()) <= 1 { + sectionPages := p1.Pages() + for i, p2 := range sectionPages { + p2s := p2.(*pageState) + if p2s.posNextPrevSection == nil { + continue + } + + p2s.posNextPrevSection.nextPage = nil + p2s.posNextPrevSection.prevPage = nil + + if i > 0 { + p2s.posNextPrevSection.nextPage = sectionPages[i-1] + } + + if i < len(sectionPages)-1 { + p2s.posNextPrevSection.prevPage = sectionPages[i+1] + } + } + } + } + + for i, j := range rootSection { + p := s.workAllPages[j] + if i > 0 { + p.posNextPrevSection.nextPage = s.workAllPages[rootSection[i-1]] + } + + if i < len(rootSection)-1 { + p.posNextPrevSection.prevPage = s.workAllPages[rootSection[i+1]] + } + } + + return nil, nil + }) + + s.init.menus = init.Branch(func() (interface{}, error) { + s.assembleMenus() + return nil, nil + }) + +} + +// Build stats for a given site. +type buildStats struct { + draftCount int + futureCount int + expiredCount int +} + +// TODO(bep) consolidate all site stats into this +func (b *buildStats) update(p page.Page) { + if p.Draft() { + b.draftCount++ + } + + if resource.IsFuture(p) { + b.futureCount++ + } + + if resource.IsExpired(p) { + b.expiredCount++ + } } type siteRenderingContext struct { output.Format } +func (s *Site) Menus() navigation.Menus { + s.init.menus.Do() + return s.menus +} + func (s *Site) initRenderFormats() { formatSet := make(map[string]bool) formats := output.Formats{} - for _, p := range s.Pages { - for _, f := range p.outputFormats { + for _, p := range s.workAllPages { + for _, f := range p.m.configuredOutputFormats { if !formatSet[f.Name] { formats = append(formats, f) formatSet[f.Name] = true @@ -182,10 +309,30 @@ func (s *Site) initRenderFormats() { } } + // Add the per kind configured output formats + for _, kind := range allKindsInPages { + if siteFormats, found := s.outputFormats[kind]; found { + for _, f := range siteFormats { + if !formatSet[f.Name] { + formats = append(formats, f) + formatSet[f.Name] = true + } + } + } + } + sort.Sort(formats) s.renderFormats = formats } +func (s *Site) GetRelatedDocsHandler() *page.RelatedDocsHandler { + return s.relatedDocsHandler +} + +func (s *Site) Language() *langs.Language { + return s.language +} + func (s *Site) isEnabled(kind string) bool { if kind == kindUnknown { panic("Unknown kind") @@ -199,19 +346,23 @@ func (s *Site) reset() *Site { layoutHandler: output.NewLayoutHandler(), disabledKinds: s.disabledKinds, titleFunc: s.titleFunc, - relatedDocsHandler: newSearchIndexHandler(s.relatedDocsHandler.cfg), + relatedDocsHandler: s.relatedDocsHandler.Clone(), siteRefLinker: s.siteRefLinker, outputFormats: s.outputFormats, rc: s.rc, outputFormatsConfig: s.outputFormatsConfig, frontmatterHandler: s.frontmatterHandler, mediaTypesConfig: s.mediaTypesConfig, - Language: s.Language, - owner: s.owner, + language: s.language, + h: s.h, publisher: s.publisher, - siteConfig: s.siteConfig, + siteConfigConfig: s.siteConfigConfig, enableInlineShortcodes: s.enableInlineShortcodes, - PageCollections: newPageCollections()} + buildStats: &buildStats{}, + init: s.init, + PageCollections: newPageCollections(), + siteCfg: s.siteCfg, + } } @@ -262,6 +413,8 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { return nil, err } + taxonomies := cfg.Language.GetStringMapString("taxonomies") + var relatedContentConfig related.Config if cfg.Language.IsSet("related") { @@ -271,7 +424,6 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { } } else { relatedContentConfig = related.DefaultConfig - taxonomies := cfg.Language.GetStringMapString("taxonomies") if _, found := taxonomies["tag"]; found { relatedContentConfig.Add(related.IndexConfig{Name: "tags", Weight: 80}) } @@ -284,21 +436,33 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { return nil, err } + siteConfig := siteConfigHolder{ + sitemap: config.DecodeSitemap(config.Sitemap{Priority: -1, Filename: "sitemap.xml"}, cfg.Language.GetStringMap("sitemap")), + taxonomiesConfig: taxonomies, + timeout: time.Duration(cfg.Language.GetInt("timeout")) * time.Millisecond, + hasCJKLanguage: cfg.Language.GetBool("hasCJKLanguage"), + enableEmoji: cfg.Language.Cfg.GetBool("enableEmoji"), + } + s := &Site{ PageCollections: c, layoutHandler: output.NewLayoutHandler(), - Language: cfg.Language, + language: cfg.Language, disabledKinds: disabledKinds, titleFunc: titleFunc, - relatedDocsHandler: newSearchIndexHandler(relatedContentConfig), + relatedDocsHandler: page.NewRelatedDocsHandler(relatedContentConfig), outputFormats: outputFormats, rc: &siteRenderingContext{output.HTMLFormat}, outputFormatsConfig: siteOutputFormatsConfig, mediaTypesConfig: siteMediaTypesConfig, frontmatterHandler: frontMatterHandler, + buildStats: &buildStats{}, enableInlineShortcodes: cfg.Language.GetBool("enableInlineShortcodes"), + siteCfg: siteConfig, } + s.prepareInits() + return s, nil } @@ -372,52 +536,94 @@ func NewSiteForCfg(cfg deps.DepsCfg) (*Site, error) { } -type SiteInfos []*SiteInfo +type SiteInfo struct { + Authors page.AuthorList + Social SiteSocial -// First is a convenience method to get the first Site, i.e. the main language. -func (s SiteInfos) First() *SiteInfo { - if len(s) == 0 { - return nil - } - return s[0] -} + hugoInfo hugo.Info + title string + RSSLink string + Author map[string]interface{} + LanguageCode string + Copyright string + + permalinks map[string]string + + LanguagePrefix string + Languages langs.Languages + + BuildDrafts bool + + canonifyURLs bool + relativeURLs bool + uglyURLs func(p page.Page) bool -type SiteInfo struct { - Taxonomies TaxonomyList - Authors AuthorList - Social SiteSocial - *PageCollections - Menus *Menus - hugoInfo hugo.Info - Title string - RSSLink string - Author map[string]interface{} - LanguageCode string - Copyright string - LastChange time.Time - Permalinks PermalinkOverrides - Params map[string]interface{} - BuildDrafts bool - canonifyURLs bool - relativeURLs bool - uglyURLs func(p *Page) bool - preserveTaxonomyNames bool - Data *map[string]interface{} owner *HugoSites s *Site language *langs.Language - LanguagePrefix string - Languages langs.Languages defaultContentLanguageInSubdir bool sectionPagesMenu string } +func (s *SiteInfo) Pages() page.Pages { + return s.s.Pages() + +} + +func (s *SiteInfo) RegularPages() page.Pages { + return s.s.RegularPages() + +} + +func (s *SiteInfo) AllPages() page.Pages { + return s.s.AllPages() +} + +func (s *SiteInfo) AllRegularPages() page.Pages { + return s.s.AllRegularPages() +} + +func (s *SiteInfo) Permalinks() map[string]string { + // Remove in 0.57 + helpers.Deprecated("Site", ".Permalinks", "", false) + return s.permalinks +} + +func (s *SiteInfo) LastChange() time.Time { + return s.s.lastmod +} + +func (s *SiteInfo) Title() string { + return s.title +} + +func (s *SiteInfo) Site() page.Site { + return s +} + +func (s *SiteInfo) Menus() navigation.Menus { + return s.s.Menus() +} + +// TODO(bep) type +func (s *SiteInfo) Taxonomies() interface{} { + return s.s.Taxonomies +} + +func (s *SiteInfo) Params() map[string]interface{} { + return s.s.Language().Params() +} + +func (s *SiteInfo) Data() map[string]interface{} { + return s.s.h.Data() +} + func (s *SiteInfo) Language() *langs.Language { return s.language } func (s *SiteInfo) Config() SiteConfig { - return s.s.siteConfig + return s.s.siteConfigConfig } func (s *SiteInfo) Hugo() hugo.Info { @@ -425,11 +631,12 @@ func (s *SiteInfo) Hugo() hugo.Info { } // Sites is a convenience method to get all the Hugo sites/languages configured. -func (s *SiteInfo) Sites() SiteInfos { - return s.s.owner.siteInfos() +func (s *SiteInfo) Sites() page.Sites { + return s.s.h.siteInfos() } + func (s *SiteInfo) String() string { - return fmt.Sprintf("Site(%q)", s.Title) + return fmt.Sprintf("Site(%q)", s.title) } func (s *SiteInfo) BaseURL() template.URL { @@ -484,7 +691,7 @@ func (s *SiteInfo) Param(key interface{}) (interface{}, error) { return nil, err } keyStr = strings.ToLower(keyStr) - return s.Params[keyStr], nil + return s.Params()[keyStr], nil } func (s *SiteInfo) IsMultiLingual() bool { @@ -513,28 +720,24 @@ func newSiteRefLinker(cfg config.Provider, s *Site) (siteRefLinker, error) { return siteRefLinker{s: s, errorLogger: logger, notFoundURL: notFoundURL}, nil } -func (s siteRefLinker) logNotFound(ref, what string, p *Page, position text.Position) { +func (s siteRefLinker) logNotFound(ref, what string, p page.Page, position text.Position) { if position.IsValid() { s.errorLogger.Printf("[%s] REF_NOT_FOUND: Ref %q: %s: %s", s.s.Lang(), ref, position.String(), what) } else if p == nil { s.errorLogger.Printf("[%s] REF_NOT_FOUND: Ref %q: %s", s.s.Lang(), ref, what) } else { - s.errorLogger.Printf("[%s] REF_NOT_FOUND: Ref %q from page %q: %s", s.s.Lang(), ref, p.pathOrTitle(), what) + s.errorLogger.Printf("[%s] REF_NOT_FOUND: Ref %q from page %q: %s", s.s.Lang(), ref, p.Path(), what) } } func (s *siteRefLinker) refLink(ref string, source interface{}, relative bool, outputFormat string) (string, error) { - var page *Page - switch v := source.(type) { - case *Page: - page = v - case pageContainer: - page = v.page() + p, err := unwrapPage(source) + if err != nil { + return "", err } var refURL *url.URL - var err error ref = filepath.ToSlash(ref) @@ -544,11 +747,11 @@ func (s *siteRefLinker) refLink(ref string, source interface{}, relative bool, o return s.notFoundURL, err } - var target *Page + var target page.Page var link string if refURL.Path != "" { - target, err := s.s.getPageNew(page, refURL.Path) + target, err := s.s.getPageNew(p, refURL.Path) var pos text.Position if err != nil || target == nil { if p, ok := source.(text.Positioner); ok { @@ -558,12 +761,12 @@ func (s *siteRefLinker) refLink(ref string, source interface{}, relative bool, o } if err != nil { - s.logNotFound(refURL.Path, err.Error(), page, pos) + s.logNotFound(refURL.Path, err.Error(), p, pos) return s.notFoundURL, nil } if target == nil { - s.logNotFound(refURL.Path, "page not found", page, pos) + s.logNotFound(refURL.Path, "page not found", p, pos) return s.notFoundURL, nil } @@ -573,7 +776,7 @@ func (s *siteRefLinker) refLink(ref string, source interface{}, relative bool, o o := target.OutputFormats().Get(outputFormat) if o == nil { - s.logNotFound(refURL.Path, fmt.Sprintf("output format %q", outputFormat), page, pos) + s.logNotFound(refURL.Path, fmt.Sprintf("output format %q", outputFormat), p, pos) return s.notFoundURL, nil } permalinker = o @@ -587,22 +790,24 @@ func (s *siteRefLinker) refLink(ref string, source interface{}, relative bool, o } if refURL.Fragment != "" { + _ = target link = link + "#" + refURL.Fragment - - if refURL.Path != "" && target != nil && !target.getRenderingConfig().PlainIDAnchors { - link = link + ":" + target.UniqueID() - } else if page != nil && !page.getRenderingConfig().PlainIDAnchors { - link = link + ":" + page.UniqueID() + if pctx, ok := target.(pageContext); ok && target.File() != nil && !pctx.getRenderingConfig().PlainIDAnchors { + if refURL.Path != "" { + link = link + ":" + target.File().UniqueID() + } + } else if pctx, ok := p.(pageContext); ok && p.File() != nil && !pctx.getRenderingConfig().PlainIDAnchors { + link = link + ":" + p.File().UniqueID() } - } + } return link, nil } // Ref will give an absolute URL to ref in the given Page. -func (s *SiteInfo) Ref(ref string, page *Page, options ...string) (string, error) { - // Remove in Hugo 0.53 - helpers.Deprecated("Site", ".Ref", "Use .Site.GetPage", false) +func (s *SiteInfo) Ref(ref string, page page.Page, options ...string) (string, error) { + // Remove in Hugo 0.54 + helpers.Deprecated("Site", ".Ref", "Use .Site.GetPage", true) outputFormat := "" if len(options) > 0 { outputFormat = options[0] @@ -612,9 +817,9 @@ func (s *SiteInfo) Ref(ref string, page *Page, options ...string) (string, error } // RelRef will give an relative URL to ref in the given Page. -func (s *SiteInfo) RelRef(ref string, page *Page, options ...string) (string, error) { - // Remove in Hugo 0.53 - helpers.Deprecated("Site", ".RelRef", "Use .Site.GetPage", false) +func (s *SiteInfo) RelRef(ref string, page page.Page, options ...string) (string, error) { + // Remove in Hugo 0.54 + helpers.Deprecated("Site", ".RelRef", "Use .Site.GetPage", true) outputFormat := "" if len(options) > 0 { outputFormat = options[0] @@ -624,22 +829,11 @@ func (s *SiteInfo) RelRef(ref string, page *Page, options ...string) (string, er } func (s *Site) running() bool { - return s.owner != nil && s.owner.running + return s.h != nil && s.h.running } func (s *Site) multilingual() *Multilingual { - return s.owner.multilingual -} - -func init() { - defaultTimer = nitro.Initalize() -} - -func (s *Site) timerStep(step string) { - if s.timer == nil { - s.timer = defaultTimer - } - s.timer.Step(step) + return s.h.multilingual } type whatChanged struct { @@ -737,9 +931,7 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { s.Log.DEBUG.Printf("Rebuild for events %q", events) - h := s.owner - - s.timerStep("initialize rebuild") + h := s.h // First we need to determine what changed @@ -771,7 +963,6 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { tmplChanged = append(tmplChanged, ev) if strings.Contains(ev.Name, "shortcodes") { - clearIsInnerShortcodeCache() shortcode := filepath.Base(ev.Name) shortcode = strings.TrimSuffix(shortcode, filepath.Ext(shortcode)) shortcodesChanged[shortcode] = true @@ -788,14 +979,16 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { } // These in memory resource caches will be rebuilt on demand. - for _, s := range s.owner.Sites { + for _, s := range s.h.Sites { s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...) } if len(tmplChanged) > 0 || len(i18nChanged) > 0 { - sites := s.owner.Sites + sites := s.h.Sites first := sites[0] + s.h.init.Reset() + // TOD(bep) globals clean if err := first.Deps.LoadResources(); err != nil { return whatChanged{}, err @@ -805,7 +998,7 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { site := sites[i] var err error depsCfg := deps.DepsCfg{ - Language: site.Language, + Language: site.language, MediaTypes: site.mediaTypesConfig, OutputFormats: site.outputFormatsConfig, } @@ -817,14 +1010,10 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { return whatChanged{}, err } } - - s.timerStep("template prep") } if len(dataChanged) > 0 { - if err := s.readDataFromSourceFS(); err != nil { - return whatChanged{}, err - } + s.h.init.data.Reset() } for _, ev := range sourceChanged { @@ -860,7 +1049,7 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { // pages that keeps a reference to the changed shortcode. pagesWithShortcode := h.findPagesByShortcode(shortcode) for _, p := range pagesWithShortcode { - contentFilesChanged = append(contentFilesChanged, p.File.Filename()) + contentFilesChanged = append(contentFilesChanged, p.File().Filename()) } } @@ -891,193 +1080,72 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { } -func (s *Site) loadData(fs afero.Fs) (err error) { - spec := src.NewSourceSpec(s.PathSpec, fs) - fileSystem := spec.NewFilesystem("") - s.Data = make(map[string]interface{}) - for _, r := range fileSystem.Files() { - if err := s.handleDataFile(r); err != nil { - return err - } +func (s *Site) process(config BuildCfg) (err error) { + if err = s.initialize(); err != nil { + return } - - return -} - -func (s *Site) errWithFileContext(err error, f source.File) error { - rfi, ok := f.FileInfo().(hugofs.RealFilenameInfo) - if !ok { + if err := s.readAndProcessContent(); err != nil { return err } - - realFilename := rfi.RealFilename() - - err, _ = herrors.WithFileContextForFile( - err, - realFilename, - realFilename, - s.SourceSpec.Fs.Source, - herrors.SimpleLineMatcher) - return err -} -func (s *Site) handleDataFile(r source.ReadableFile) error { - var current map[string]interface{} - - f, err := r.Open() - if err != nil { - return _errors.Wrapf(err, "Failed to open data file %q:", r.LogicalName()) - } - defer f.Close() - - // Crawl in data tree to insert data - current = s.Data - keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator) - // The first path element is the virtual folder (typically theme name), which is - // not part of the key. - if len(keyParts) > 1 { - for _, key := range keyParts[1:] { - if key != "" { - if _, ok := current[key]; !ok { - current[key] = make(map[string]interface{}) - } - current = current[key].(map[string]interface{}) - } - } - } - - data, err := s.readData(r) - if err != nil { - return s.errWithFileContext(err, r) - } - - if data == nil { - return nil - } - - // filepath.Walk walks the files in lexical order, '/' comes before '.' - // this warning could happen if - // 1. A theme uses the same key; the main data folder wins - // 2. A sub folder uses the same key: the sub folder wins - higherPrecedentData := current[r.BaseFileName()] - - switch data.(type) { - case nil: - // hear the crickets? - - case map[string]interface{}: - - switch higherPrecedentData.(type) { - case nil: - current[r.BaseFileName()] = data - case map[string]interface{}: - // merge maps: insert entries from data for keys that - // don't already exist in higherPrecedentData - higherPrecedentMap := higherPrecedentData.(map[string]interface{}) - for key, value := range data.(map[string]interface{}) { - if _, exists := higherPrecedentMap[key]; exists { - s.Log.WARN.Printf("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path()) - } else { - higherPrecedentMap[key] = value - } - } - default: - // can't merge: higherPrecedentData is not a map - s.Log.WARN.Printf("The %T data from '%s' overridden by "+ - "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData) - } - - case []interface{}: - if higherPrecedentData == nil { - current[r.BaseFileName()] = data - } else { - // we don't merge array data - s.Log.WARN.Printf("The %T data from '%s' overridden by "+ - "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData) - } - - default: - s.Log.ERROR.Printf("unexpected data type %T in file %s", data, r.LogicalName()) - } - - return nil } -func (s *Site) readData(f source.ReadableFile) (interface{}, error) { - file, err := f.Open() - if err != nil { - return nil, _errors.Wrap(err, "readData: failed to open data file") +func (s *Site) setupSitePages() { + var homeDates *resource.Dates + if s.home != nil { + // If the home page has no dates set, we fall back to the site dates. + homeDates = &s.home.m.Dates } - defer file.Close() - content := helpers.ReaderToBytes(file) - - format := metadecoders.FormatFromString(f.Extension()) - return metadecoders.Default.Unmarshal(content, format) -} -func (s *Site) readDataFromSourceFS() error { - err := s.loadData(s.PathSpec.BaseFs.Data.Fs) - s.timerStep("load data") - return err -} - -func (s *Site) process(config BuildCfg) (err error) { - if err = s.initialize(); err != nil { + if !s.lastmod.IsZero() && (homeDates == nil || !resource.IsZeroDates(homeDates)) { return } - s.timerStep("initialize") - if err = s.readDataFromSourceFS(); err != nil { + if homeDates != nil && !s.lastmod.IsZero() { + homeDates.FDate = s.lastmod + homeDates.FLastmod = s.lastmod return - } - - s.timerStep("load i18n") - if err := s.readAndProcessContent(); err != nil { - return err } - s.timerStep("read and convert pages from source") - return err + var siteLastmod time.Time + var siteLastDate time.Time -} - -func (s *Site) setupSitePages() { - var siteLastChange time.Time - - for i, page := range s.RegularPages { - if i > 0 { - page.NextPage = s.RegularPages[i-1] - } - - if i < len(s.RegularPages)-1 { - page.PrevPage = s.RegularPages[i+1] + for _, page := range s.workAllPages { + if !page.IsPage() { + continue } - // Determine Site.Info.LastChange // Note that the logic to determine which date to use for Lastmod // is already applied, so this is *the* date to use. // We cannot just pick the last page in the default sort, because // that may not be ordered by date. - if page.Lastmod.After(siteLastChange) { - siteLastChange = page.Lastmod + // TODO(bep) check if this can be done earlier + if page.Lastmod().After(siteLastmod) { + siteLastmod = page.Lastmod() } + if page.Date().After(siteLastDate) { + siteLastDate = page.Date() + } + } + + s.lastmod = siteLastmod + + if homeDates != nil && resource.IsZeroDates(homeDates) { + homeDates.FDate = siteLastDate + homeDates.FLastmod = s.lastmod } - s.Info.LastChange = siteLastChange } -func (s *Site) render(config *BuildCfg, outFormatIdx int) (err error) { - // Clear the global page cache. - spc.clear() +func (s *Site) render(ctx *siteRenderContext) (err error) { - if outFormatIdx == 0 { - if err = s.preparePages(); err != nil { - return - } - s.timerStep("prepare pages") + if err := page.Clear(); err != nil { + return err + } + if ctx.outIdx == 0 { // Note that even if disableAliases is set, the aliases themselves are // preserved on page. The motivation with this is to be able to generate // 301 redirects in a .htacess file and similar using a custom output format. @@ -1089,36 +1157,35 @@ func (s *Site) render(config *BuildCfg, outFormatIdx int) (err error) { if err = s.renderAliases(); err != nil { return } - s.timerStep("render and write aliases") } } - if err = s.renderPages(config); err != nil { + if err = s.renderPages(ctx); err != nil { return } - s.timerStep("render and write pages") + if ctx.outIdx == 0 { + if err = s.renderSitemap(); err != nil { + return + } - // TODO(bep) render consider this, ref. render404 etc. - if outFormatIdx > 0 { - return - } + if err = s.renderRobotsTXT(); err != nil { + return + } - if err = s.renderSitemap(); err != nil { - return + if err = s.render404(); err != nil { + return + } } - s.timerStep("render and write Sitemap") - if err = s.renderRobotsTXT(); err != nil { + if !ctx.renderSingletonPages() { return } - s.timerStep("render and write robots.txt") - if err = s.render404(); err != nil { + if err = s.renderMainLanguageRedirect(); err != nil { return } - s.timerStep("render and write 404") return } @@ -1128,8 +1195,6 @@ func (s *Site) Initialise() (err error) { } func (s *Site) initialize() (err error) { - s.Menus = Menus{} - return s.initializeSiteInfo() } @@ -1144,31 +1209,25 @@ func (s *SiteInfo) HomeAbsURL() string { // SitemapAbsURL is a convenience method giving the absolute URL to the sitemap. func (s *SiteInfo) SitemapAbsURL() string { - sitemapDefault := parseSitemap(s.s.Cfg.GetStringMap("sitemap")) p := s.HomeAbsURL() if !strings.HasSuffix(p, "/") { p += "/" } - p += sitemapDefault.Filename + p += s.s.siteCfg.sitemap.Filename return p } func (s *Site) initializeSiteInfo() error { var ( - lang = s.Language + lang = s.language languages langs.Languages ) - if s.owner != nil && s.owner.multilingual != nil { - languages = s.owner.multilingual.Languages + if s.h != nil && s.h.multilingual != nil { + languages = s.h.multilingual.Languages } - params := lang.Params() - - permalinks := make(PermalinkOverrides) - for k, v := range s.Cfg.GetStringMapString("permalinks") { - permalinks[k] = pathPattern(v) - } + permalinks := s.Cfg.GetStringMapString("permalinks") defaultContentInSubDir := s.Cfg.GetBool("defaultContentLanguageInSubdir") defaultContentLanguage := s.Cfg.GetString("defaultContentLanguage") @@ -1178,7 +1237,7 @@ func (s *Site) initializeSiteInfo() error { languagePrefix = "/" + lang.Lang } - var uglyURLs = func(p *Page) bool { + var uglyURLs = func(p page.Page) bool { return false } @@ -1186,25 +1245,25 @@ func (s *Site) initializeSiteInfo() error { if v != nil { switch vv := v.(type) { case bool: - uglyURLs = func(p *Page) bool { + uglyURLs = func(p page.Page) bool { return vv } case string: // Is what be get from CLI (--uglyURLs) vvv := cast.ToBool(vv) - uglyURLs = func(p *Page) bool { + uglyURLs = func(p page.Page) bool { return vvv } default: m := cast.ToStringMapBool(v) - uglyURLs = func(p *Page) bool { + uglyURLs = func(p page.Page) bool { return m[p.Section()] } } } s.Info = SiteInfo{ - Title: lang.GetString("title"), + title: lang.GetString("title"), Author: lang.GetStringMap("author"), Social: lang.GetStringMapString("social"), LanguageCode: lang.GetString("languageCode"), @@ -1218,20 +1277,13 @@ func (s *Site) initializeSiteInfo() error { canonifyURLs: s.Cfg.GetBool("canonifyURLs"), relativeURLs: s.Cfg.GetBool("relativeURLs"), uglyURLs: uglyURLs, - preserveTaxonomyNames: lang.GetBool("preserveTaxonomyNames"), - PageCollections: s.PageCollections, - Menus: &s.Menus, - Params: params, - Permalinks: permalinks, - Data: &s.Data, - owner: s.owner, + permalinks: permalinks, + owner: s.h, s: s, hugoInfo: hugo.NewInfo(s.Cfg.GetString("environment")), - // TODO(bep) make this Menu and similar into delegate methods on SiteInfo - Taxonomies: s.Taxonomies, } - rssOutputFormat, found := s.outputFormats[KindHome].GetByName(output.RSSFormat.Name) + rssOutputFormat, found := s.outputFormats[page.KindHome].GetByName(output.RSSFormat.Name) if found { s.Info.RSSLink = s.permalink(rssOutputFormat.BaseFilename()) @@ -1252,10 +1304,6 @@ func (s *Site) isLayoutDirEvent(e fsnotify.Event) bool { return s.BaseFs.SourceFilesystems.IsLayout(e.Name) } -func (s *Site) absContentDir() string { - return s.PathSpec.AbsPathify(s.PathSpec.ContentDir) -} - func (s *Site) isContentDirEvent(e fsnotify.Event) bool { return s.BaseFs.IsContent(e.Name) } @@ -1286,13 +1334,13 @@ func (c *contentCaptureResultHandler) handleBundles(d *bundleDirs) { } } -func (c *contentCaptureResultHandler) handleCopyFiles(files ...pathLangFile) { - for _, proc := range c.contentProcessors { - proc.processAssets(files) - } +func (c *contentCaptureResultHandler) handleCopyFile(f pathLangFile) { + proc := c.getContentProcessor(f.Lang()) + proc.processAsset(f) } func (s *Site) readAndProcessContent(filenames ...string) error { + ctx := context.Background() g, ctx := errgroup.WithContext(ctx) @@ -1300,9 +1348,9 @@ func (s *Site) readAndProcessContent(filenames ...string) error { contentProcessors := make(map[string]*siteContentProcessor) var defaultContentProcessor *siteContentProcessor - sites := s.owner.langSite() + sites := s.h.langSite() for k, v := range sites { - if v.Language.Disabled { + if v.language.Disabled { continue } proc := newSiteContentProcessor(ctx, len(filenames) > 0, v) @@ -1326,7 +1374,7 @@ func (s *Site) readAndProcessContent(filenames ...string) error { if s.running() { // Need to track changes. - bundleMap = s.owner.ContentChanges + bundleMap = s.h.ContentChanges handler = &captureResultHandlerChain{handlers: []captureBundlesHandler{mainHandler, bundleMap}} } else { @@ -1349,28 +1397,11 @@ func (s *Site) readAndProcessContent(filenames ...string) error { return err2 } -func (s *Site) buildSiteMeta() (err error) { - defer s.timerStep("build Site meta") - - if len(s.Pages) == 0 { - return - } - - s.assembleTaxonomies() - - for _, p := range s.AllPages { - // this depends on taxonomies - p.setValuesForKind(s) - } - - return -} - -func (s *Site) getMenusFromConfig() Menus { +func (s *Site) getMenusFromConfig() navigation.Menus { - ret := Menus{} + ret := navigation.Menus{} - if menus := s.Language.GetStringMap("menus"); menus != nil { + if menus := s.language.GetStringMap("menus"); menus != nil { for name, menu := range menus { m, err := cast.ToSliceE(menu) if err != nil { @@ -1380,20 +1411,20 @@ func (s *Site) getMenusFromConfig() Menus { for _, entry := range m { s.Log.DEBUG.Printf("found menu: %q, in site config\n", name) - menuEntry := MenuEntry{Menu: name} + menuEntry := navigation.MenuEntry{Menu: name} ime, err := cast.ToStringMapE(entry) if err != nil { s.Log.ERROR.Printf("unable to process menus in site config\n") s.Log.ERROR.Println(err) } - menuEntry.marshallMap(ime) + menuEntry.MarshallMap(ime) menuEntry.URL = s.Info.createNodeMenuEntryURL(menuEntry.URL) if ret[name] == nil { - ret[name] = &Menu{} + ret[name] = navigation.Menu{} } - *ret[name] = ret[name].add(&menuEntry) + ret[name] = ret[name].Add(&menuEntry) } } } @@ -1417,28 +1448,27 @@ func (s *SiteInfo) createNodeMenuEntryURL(in string) string { } func (s *Site) assembleMenus() { - s.Menus = Menus{} + s.menus = make(navigation.Menus) type twoD struct { MenuName, EntryName string } - flat := map[twoD]*MenuEntry{} - children := map[twoD]Menu{} + flat := map[twoD]*navigation.MenuEntry{} + children := map[twoD]navigation.Menu{} // add menu entries from config to flat hash menuConfig := s.getMenusFromConfig() for name, menu := range menuConfig { - for _, me := range *menu { + for _, me := range menu { flat[twoD{name, me.KeyName()}] = me } } sectionPagesMenu := s.Info.sectionPagesMenu - pages := s.Pages if sectionPagesMenu != "" { - for _, p := range pages { - if p.Kind == KindSection { + for _, p := range s.workAllPages { + if p.Kind() == page.KindSection { // From Hugo 0.22 we have nested sections, but until we get a // feel of how that would work in this setting, let us keep // this menu for the top level only. @@ -1447,9 +1477,9 @@ func (s *Site) assembleMenus() { continue } - me := MenuEntry{Identifier: id, + me := navigation.MenuEntry{Identifier: id, Name: p.LinkTitle(), - Weight: p.Weight, + Weight: p.Weight(), URL: p.RelPermalink()} flat[twoD{sectionPagesMenu, me.KeyName()}] = &me } @@ -1457,10 +1487,10 @@ func (s *Site) assembleMenus() { } // Add menu entries provided by pages - for _, p := range pages { - for name, me := range p.Menus() { + for _, p := range s.workAllPages { + for name, me := range p.pageMenus.menus() { if _, ok := flat[twoD{name, me.KeyName()}]; ok { - s.SendError(p.errWithFileContext(errors.Errorf("duplicate menu entry with identifier %q in menu %q", me.KeyName(), name))) + s.SendError(p.wrapError(errors.Errorf("duplicate menu entry with identifier %q in menu %q", me.KeyName(), name))) continue } flat[twoD{name, me.KeyName()}] = me @@ -1470,7 +1500,7 @@ func (s *Site) assembleMenus() { // Create Children Menus First for _, e := range flat { if e.Parent != "" { - children[twoD{e.Menu, e.Parent}] = children[twoD{e.Menu, e.Parent}].add(e) + children[twoD{e.Menu, e.Parent}] = children[twoD{e.Menu, e.Parent}].Add(e) } } @@ -1479,7 +1509,7 @@ func (s *Site) assembleMenus() { _, ok := flat[twoD{p.MenuName, p.EntryName}] if !ok { // if parent does not exist, create one without a URL - flat[twoD{p.MenuName, p.EntryName}] = &MenuEntry{Name: p.EntryName, URL: ""} + flat[twoD{p.MenuName, p.EntryName}] = &navigation.MenuEntry{Name: p.EntryName, URL: ""} } flat[twoD{p.MenuName, p.EntryName}].Children = childmenu } @@ -1487,122 +1517,127 @@ func (s *Site) assembleMenus() { // Assembling Top Level of Tree for menu, e := range flat { if e.Parent == "" { - _, ok := s.Menus[menu.MenuName] + _, ok := s.menus[menu.MenuName] if !ok { - s.Menus[menu.MenuName] = &Menu{} + s.menus[menu.MenuName] = navigation.Menu{} } - *s.Menus[menu.MenuName] = s.Menus[menu.MenuName].add(e) + s.menus[menu.MenuName] = s.menus[menu.MenuName].Add(e) } } } +// get any lanaguagecode to prefix the target file path with. +func (s *Site) getLanguageTargetPathLang(alwaysInSubDir bool) string { + if s.h.IsMultihost() { + return s.Language().Lang + } + + return s.getLanguagePermalinkLang(alwaysInSubDir) +} + +// get any lanaguagecode to prefix the relative permalink with. +func (s *Site) getLanguagePermalinkLang(alwaysInSubDir bool) string { + + if !s.Info.IsMultiLingual() || s.h.IsMultihost() { + return "" + } + + if alwaysInSubDir { + return s.Language().Lang + } + + isDefault := s.Language().Lang == s.multilingual().DefaultLang.Lang + + if !isDefault || s.Info.defaultContentLanguageInSubdir { + return s.Language().Lang + } + + return "" +} + func (s *Site) getTaxonomyKey(key string) string { - if s.Info.preserveTaxonomyNames { - // Keep as is - return key + if s.PathSpec.DisablePathToLower { + return s.PathSpec.MakePath(key) } - return s.PathSpec.MakePathSanitized(key) + return strings.ToLower(s.PathSpec.MakePath(key)) } -// We need to create the top level taxonomy early in the build process -// to be able to determine the page Kind correctly. -func (s *Site) createTaxonomiesEntries() { +func (s *Site) assembleTaxonomies() error { s.Taxonomies = make(TaxonomyList) - taxonomies := s.Language.GetStringMapString("taxonomies") + taxonomies := s.siteCfg.taxonomiesConfig for _, plural := range taxonomies { s.Taxonomies[plural] = make(Taxonomy) } -} -func (s *Site) assembleTaxonomies() { - s.taxonomiesPluralSingular = make(map[string]string) - s.taxonomiesOrigKey = make(map[string]string) - - taxonomies := s.Language.GetStringMapString("taxonomies") + s.taxonomyNodes = make(taxonomyNodeInfos) s.Log.INFO.Printf("found taxonomies: %#v\n", taxonomies) for singular, plural := range taxonomies { - s.taxonomiesPluralSingular[plural] = singular + parent := s.taxonomyNodes.GetOrCreate(plural, "", "") + parent.singular = singular + + addTaxonomy := func(plural, term string, weight int, p page.Page) { + key := s.getTaxonomyKey(term) + + n := s.taxonomyNodes.GetOrCreate(plural, key, term) + n.parent = parent - for _, p := range s.Pages { - vals := p.getParam(plural, !s.Info.preserveTaxonomyNames) + // There may be different spellings before normalization, so the + // last one will win, e.g. "hugo" vs "Hugo". + n.term = term - w := p.getParamToLower(plural + "_weight") + w := page.NewWeightedPage(weight, p, n.getOwner) + + s.Taxonomies[plural].add(key, w) + + n.UpdateFromPage(w.Page) + parent.UpdateFromPage(w.Page) + } + + for _, p := range s.workAllPages { + vals := getParam(p, plural, false) + + w := getParamToLower(p, plural+"_weight") weight, err := cast.ToIntE(w) if err != nil { - s.Log.ERROR.Printf("Unable to convert taxonomy weight %#v to int for %s", w, p.File.Path()) + s.Log.ERROR.Printf("Unable to convert taxonomy weight %#v to int for %q", w, p.pathOrTitle()) // weight will equal zero, so let the flow continue } if vals != nil { if v, ok := vals.([]string); ok { for _, idx := range v { - x := WeightedPage{weight, p} - s.Taxonomies[plural].add(s.getTaxonomyKey(idx), x) - if s.Info.preserveTaxonomyNames { - // Need to track the original - s.taxonomiesOrigKey[fmt.Sprintf("%s-%s", plural, s.PathSpec.MakePathSanitized(idx))] = idx - } + addTaxonomy(plural, idx, weight, p) } } else if v, ok := vals.(string); ok { - x := WeightedPage{weight, p} - s.Taxonomies[plural].add(s.getTaxonomyKey(v), x) - if s.Info.preserveTaxonomyNames { - // Need to track the original - s.taxonomiesOrigKey[fmt.Sprintf("%s-%s", plural, s.PathSpec.MakePathSanitized(v))] = v - } + addTaxonomy(plural, v, weight, p) } else { - s.Log.ERROR.Printf("Invalid %s in %s\n", plural, p.File.Path()) + s.Log.ERROR.Printf("Invalid %s in %q\n", plural, p.pathOrTitle()) } } } + for k := range s.Taxonomies[plural] { s.Taxonomies[plural][k].Sort() } } - s.Info.Taxonomies = s.Taxonomies + return nil } // Prepare site for a new full build. func (s *Site) resetBuildState() { - - s.relatedDocsHandler = newSearchIndexHandler(s.relatedDocsHandler.cfg) + s.relatedDocsHandler = s.relatedDocsHandler.Clone() s.PageCollections = newPageCollectionsFromPages(s.rawAllPages) - // TODO(bep) get rid of this double - s.Info.PageCollections = s.PageCollections - - s.draftCount = 0 - s.futureCount = 0 - - s.expiredCount = 0 + s.buildStats = &buildStats{} + s.init.Reset() for _, p := range s.rawAllPages { - p.subSections = Pages{} + p.subSections = page.Pages{} p.parent = nil - p.scratch = maps.NewScratch() - p.mainPageOutput = nil - } -} - -func (s *Site) layouts(p *PageOutput) ([]string, error) { - return s.layoutHandler.For(p.layoutDescriptor, p.outputFormat) -} - -func (s *Site) preparePages() error { - var errors []error - - for _, p := range s.Pages { - if err := p.prepareLayouts(); err != nil { - errors = append(errors, err) - } - if err := p.prepareData(s); err != nil { - errors = append(errors, err) - } + p.Scratcher = maps.NewScratcher() } - - return s.owner.pickOneAndLogTheRest(errors) } func (s *Site) errorCollator(results <-chan error, errs chan<- error) { @@ -1611,7 +1646,7 @@ func (s *Site) errorCollator(results <-chan error, errs chan<- error) { errors = append(errors, e) } - errs <- s.owner.pickOneAndLogTheRest(errors) + errs <- s.h.pickOneAndLogTheRest(errors) close(errs) } @@ -1623,25 +1658,17 @@ func (s *Site) errorCollator(results <-chan error, errs chan<- error) { // When we now remove the Kind from this API, we need to make the transition as painless // as possible for existing sites. Most sites will use {{ .Site.GetPage "section" "my/section" }}, // i.e. 2 arguments, so we test for that. -func (s *SiteInfo) GetPage(ref ...string) (*Page, error) { - return s.getPageOldVersion(ref...) -} +func (s *SiteInfo) GetPage(ref ...string) (page.Page, error) { + p, err := s.s.getPageOldVersion(ref...) -func (s *Site) permalinkForOutputFormat(link string, f output.Format) (string, error) { - var ( - baseURL string - err error - ) - - if f.Protocol != "" { - baseURL, err = s.PathSpec.BaseURL.WithProtocol(f.Protocol) - if err != nil { - return "", err - } - } else { - baseURL = s.PathSpec.BaseURL.String() + if p == nil { + // The nil struct has meaning in some situations, mostly to avoid breaking + // existing sites doing $nilpage.IsDescendant($p), which will always return + // false. + p = page.NilPage } - return s.PathSpec.PermalinkForBaseURL(link, baseURL), nil + + return p, err } func (s *Site) permalink(link string) string { @@ -1653,9 +1680,8 @@ func (s *Site) renderAndWriteXML(statCounter *uint64, name string, targetPath st s.Log.DEBUG.Printf("Render XML for %q to %q", name, targetPath) renderBuffer := bp.GetBuffer() defer bp.PutBuffer(renderBuffer) - renderBuffer.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>\n") - if err := s.renderForLayouts(name, d, renderBuffer, layouts...); err != nil { + if err := s.renderForLayouts(name, "", d, renderBuffer, layouts...); err != nil { return err } @@ -1684,12 +1710,13 @@ func (s *Site) renderAndWriteXML(statCounter *uint64, name string, targetPath st } -func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath string, p *PageOutput, layouts ...string) error { +func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath string, p *pageState, layouts ...string) error { renderBuffer := bp.GetBuffer() defer bp.PutBuffer(renderBuffer) - if err := s.renderForLayouts(p.Kind, p, renderBuffer, layouts...); err != nil { + of := p.outputFormat() + if err := s.renderForLayouts(p.Kind(), of.Name, p, renderBuffer, layouts...); err != nil { return err } @@ -1697,13 +1724,14 @@ func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath s return nil } - isHTML := p.outputFormat.IsHTML + isHTML := of.IsHTML + isRSS := of.Name == "RSS" var path string if s.Info.relativeURLs { path = helpers.GetDottedRelativePath(targetPath) - } else if s.Info.canonifyURLs { + } else if isRSS || s.Info.canonifyURLs { url := s.PathSpec.BaseURL.String() if !strings.HasSuffix(url, "/") { url += "/" @@ -1715,10 +1743,13 @@ func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath s Src: renderBuffer, TargetPath: targetPath, StatCounter: statCounter, - OutputFormat: p.outputFormat, + OutputFormat: p.outputFormat(), } - if isHTML { + if isRSS { + // Always canonify URLs in RSS + pd.AbsURLPath = path + } else if isHTML { if s.Info.relativeURLs || s.Info.canonifyURLs { pd.AbsURLPath = path } @@ -1742,21 +1773,30 @@ var infoOnMissingLayout = map[string]bool{ "404": true, } -func (s *Site) renderForLayouts(name string, d interface{}, w io.Writer, layouts ...string) (err error) { - var templ tpl.Template - - templ = s.findFirstTemplate(layouts...) +func (s *Site) renderForLayouts(name, outputFormat string, d interface{}, w io.Writer, layouts ...string) (err error) { + templ := s.findFirstTemplate(layouts...) if templ == nil { log := s.Log.WARN if infoOnMissingLayout[name] { log = s.Log.INFO } - if p, ok := d.(*PageOutput); ok { - log.Printf("Found no layout for %q, language %q, output format %q: create a template below /layouts with one of these filenames: %s\n", name, s.Language.Lang, p.outputFormat.Name, layoutsLogFormat(layouts)) - } else { - log.Printf("Found no layout for %q, language %q: create a template below /layouts with one of these filenames: %s\n", name, s.Language.Lang, layoutsLogFormat(layouts)) + errMsg := "You should create a template file which matches Hugo Layouts Lookup Rules for this combination." + var args []interface{} + msg := "found no layout file for" + if outputFormat != "" { + msg += " %q" + args = append(args, outputFormat) } + if name != "" { + msg += " for %q" + args = append(args, name) + } + + msg += ": " + errMsg + + log.Printf(msg, args...) + return nil } @@ -1766,20 +1806,6 @@ func (s *Site) renderForLayouts(name string, d interface{}, w io.Writer, layouts return } -func layoutsLogFormat(layouts []string) string { - var filtered []string - for _, l := range layouts { - // This is a technical prefix of no interest to the user. - lt := strings.TrimPrefix(l, "_text/") - // We have this in the lookup path for historical reasons. - lt = strings.TrimPrefix(lt, "page/") - filtered = append(filtered, lt) - } - - filtered = helpers.UniqueStrings(filtered) - return strings.Join(filtered, ", ") -} - func (s *Site) findFirstTemplate(layouts ...string) tpl.Template { for _, layout := range layouts { if templ, found := s.Tmpl.Lookup(layout); found { @@ -1795,69 +1821,93 @@ func (s *Site) publish(statCounter *uint64, path string, r io.Reader) (err error return helpers.WriteToDisk(filepath.Clean(path), r, s.BaseFs.PublishFs) } -func getGoMaxProcs() int { - if gmp := os.Getenv("GOMAXPROCS"); gmp != "" { - if p, err := strconv.Atoi(gmp); err != nil { - return p +func (s *Site) kindFromFileInfoOrSections(fi *fileInfo, sections []string) string { + if fi.TranslationBaseName() == "_index" { + if fi.Dir() == "" { + return page.KindHome } + + return s.kindFromSections(sections) + } - return 1 + return page.KindPage } -func (s *Site) newNodePage(typ string, sections ...string) *Page { - p := &Page{ - language: s.Language, - pageInit: &pageInit{}, - pageContentInit: &pageContentInit{}, - Kind: typ, - File: &source.FileInfo{}, - data: make(map[string]interface{}), - Site: &s.Info, - sections: sections, - s: s} +func (s *Site) kindFromSections(sections []string) string { + if len(sections) == 0 || len(s.siteCfg.taxonomiesConfig) == 0 { + return page.KindSection + } - p.outputFormats = p.s.outputFormats[p.Kind] + sectionPath := path.Join(sections...) - return p + for _, plural := range s.siteCfg.taxonomiesConfig { + if plural == sectionPath { + return page.KindTaxonomyTerm + } -} + if strings.HasPrefix(sectionPath, plural) { + return page.KindTaxonomy + } -func (s *Site) newHomePage() *Page { - p := s.newNodePage(KindHome) - p.title = s.Info.Title - pages := Pages{} - p.data["Pages"] = pages - p.Pages = pages - return p -} + } -func (s *Site) newTaxonomyPage(plural, key string) *Page { + return page.KindSection +} - p := s.newNodePage(KindTaxonomy, plural, key) +func (s *Site) newTaxonomyPage(title string, sections ...string) *pageState { + p, err := newPageFromMeta(&pageMeta{ + title: title, + s: s, + kind: page.KindTaxonomy, + sections: sections, + }) - if s.Info.preserveTaxonomyNames { - p.title = key - } else { - p.title = strings.Replace(s.titleFunc(key), "-", " ", -1) + if err != nil { + panic(err) } return p + } -func (s *Site) newSectionPage(name string) *Page { - p := s.newNodePage(KindSection, name) +func (s *Site) newPage(kind string, sections ...string) *pageState { + p, err := newPageFromMeta(&pageMeta{ + s: s, + kind: kind, + sections: sections, + }) - sectionName := helpers.FirstUpper(name) - if s.Cfg.GetBool("pluralizeListTitles") { - p.title = inflect.Pluralize(sectionName) - } else { - p.title = sectionName + if err != nil { + panic(err) } + return p } -func (s *Site) newTaxonomyTermsPage(plural string) *Page { - p := s.newNodePage(KindTaxonomyTerm, plural) - p.title = s.titleFunc(plural) - return p +func getGoMaxProcs() int { + if gmp := os.Getenv("GOMAXPROCS"); gmp != "" { + if p, err := strconv.Atoi(gmp); err != nil { + return p + } + } + return 1 +} + +func (s *Site) shouldBuild(p page.Page) bool { + return shouldBuild(s.BuildFuture, s.BuildExpired, + s.BuildDrafts, p.Draft(), p.PublishDate(), p.ExpiryDate()) +} + +func shouldBuild(buildFuture bool, buildExpired bool, buildDrafts bool, Draft bool, + publishDate time.Time, expiryDate time.Time) bool { + if !(buildDrafts || !Draft) { + return false + } + if !buildFuture && !publishDate.IsZero() && publishDate.After(time.Now()) { + return false + } + if !buildExpired && !expiryDate.IsZero() && expiryDate.Before(time.Now()) { + return false + } + return true } diff --git a/hugolib/siteJSONEncode_test.go b/hugolib/siteJSONEncode_test.go index 5bb6e52e8..9187751fb 100644 --- a/hugolib/siteJSONEncode_test.go +++ b/hugolib/siteJSONEncode_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,12 +14,7 @@ package hugolib import ( - "encoding/json" "testing" - - "path/filepath" - - "github.com/gohugoio/hugo/deps" ) // Issue #1123 @@ -27,27 +22,22 @@ import ( // May be smart to run with: -timeout 4000ms func TestEncodePage(t *testing.T) { t.Parallel() - cfg, fs := newTestCfg() - writeSource(t, fs, filepath.Join("content", "page.md"), `--- -title: Simple + templ := `{{ index .Site.RegularPages 0 | jsonify }}` + + b := newTestSitesBuilder(t) + b.WithSimpleConfigFile().WithTemplatesAdded("index.html", templ) + b.WithContent("page.md", `--- +title: "Page" +date: 2019-02-28 --- -Summary text -<!--more--> -`) +Content. - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) +`) - _, err := json.Marshal(s) - check(t, err) + b.Build(BuildCfg{}) - _, err = json.Marshal(s.RegularPages[0]) - check(t, err) -} + b.AssertFileContent("public/index.html", `"Date":"2019-02-28T00:00:00Z"`) -func check(t *testing.T, err error) { - if err != nil { - t.Fatalf("Failed %s", err) - } } diff --git a/hugolib/site_output.go b/hugolib/site_output.go index 0a7513961..9fb236506 100644 --- a/hugolib/site_output.go +++ b/hugolib/site_output.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import ( "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/resources/page" "github.com/spf13/cast" ) @@ -28,11 +29,11 @@ func createDefaultOutputFormats(allFormats output.Formats, cfg config.Provider) sitemapOut, _ := allFormats.GetByName(output.SitemapFormat.Name) return map[string]output.Formats{ - KindPage: {htmlOut}, - KindHome: {htmlOut, rssOut}, - KindSection: {htmlOut, rssOut}, - KindTaxonomy: {htmlOut, rssOut}, - KindTaxonomyTerm: {htmlOut, rssOut}, + page.KindPage: {htmlOut}, + page.KindHome: {htmlOut, rssOut}, + page.KindSection: {htmlOut, rssOut}, + page.KindTaxonomy: {htmlOut, rssOut}, + page.KindTaxonomyTerm: {htmlOut, rssOut}, // Below are for conistency. They are currently not used during rendering. kindRSS: {rssOut}, kindSitemap: {sitemapOut}, @@ -65,7 +66,7 @@ func createSiteOutputFormats(allFormats output.Formats, cfg config.Provider) (ma for _, format := range vals { f, found := allFormats.GetByName(format) if !found { - return nil, fmt.Errorf("Failed to resolve output format %q from site config", format) + return nil, fmt.Errorf("failed to resolve output format %q from site config", format) } formats = append(formats, f) } diff --git a/hugolib/site_output_test.go b/hugolib/site_output_test.go index e9a7e113e..e4947e5cd 100644 --- a/hugolib/site_output_test.go +++ b/hugolib/site_output_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,6 +17,8 @@ import ( "strings" "testing" + "github.com/gohugoio/hugo/resources/page" + "github.com/spf13/afero" "github.com/stretchr/testify/require" @@ -148,15 +150,15 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P require.NoError(t, err) s := h.Sites[0] - require.Equal(t, "en", s.Language.Lang) + require.Equal(t, "en", s.language.Lang) - home := s.getPage(KindHome) + home := s.getPage(page.KindHome) require.NotNil(t, home) lenOut := len(outputs) - require.Len(t, home.outputFormats, lenOut) + require.Len(t, home.OutputFormats(), lenOut) // There is currently always a JSON output to make it simpler ... altFormats := lenOut - 1 @@ -207,12 +209,8 @@ Len Pages: {{ .Kind }} {{ len .Site.RegularPages }} Page Number: {{ .Paginator.P } of := home.OutputFormats() - require.Len(t, of, lenOut) - require.Nil(t, of.Get("Hugo")) - require.NotNil(t, of.Get("json")) + json := of.Get("JSON") - _, err = home.AlternativeOutputFormats() - require.Error(t, err) require.NotNil(t, json) require.Equal(t, "/blog/index.json", json.RelPermalink()) require.Equal(t, "http://example.com/blog/index.json", json.Permalink()) @@ -323,7 +321,7 @@ baseName = "customdelimbase" th.assertFileContent("public/customdelimbase_del", "custom delim") s := h.Sites[0] - home := s.getPage(KindHome) + home := s.getPage(page.KindHome) require.NotNil(t, home) outputs := home.OutputFormats() @@ -339,8 +337,8 @@ func TestCreateSiteOutputFormats(t *testing.T) { assert := require.New(t) outputsConfig := map[string]interface{}{ - KindHome: []string{"HTML", "JSON"}, - KindSection: []string{"JSON"}, + page.KindHome: []string{"HTML", "JSON"}, + page.KindSection: []string{"JSON"}, } cfg := viper.New() @@ -348,13 +346,13 @@ func TestCreateSiteOutputFormats(t *testing.T) { outputs, err := createSiteOutputFormats(output.DefaultFormats, cfg) assert.NoError(err) - assert.Equal(output.Formats{output.JSONFormat}, outputs[KindSection]) - assert.Equal(output.Formats{output.HTMLFormat, output.JSONFormat}, outputs[KindHome]) + assert.Equal(output.Formats{output.JSONFormat}, outputs[page.KindSection]) + assert.Equal(output.Formats{output.HTMLFormat, output.JSONFormat}, outputs[page.KindHome]) // Defaults - assert.Equal(output.Formats{output.HTMLFormat, output.RSSFormat}, outputs[KindTaxonomy]) - assert.Equal(output.Formats{output.HTMLFormat, output.RSSFormat}, outputs[KindTaxonomyTerm]) - assert.Equal(output.Formats{output.HTMLFormat}, outputs[KindPage]) + assert.Equal(output.Formats{output.HTMLFormat, output.RSSFormat}, outputs[page.KindTaxonomy]) + assert.Equal(output.Formats{output.HTMLFormat, output.RSSFormat}, outputs[page.KindTaxonomyTerm]) + assert.Equal(output.Formats{output.HTMLFormat}, outputs[page.KindPage]) // These aren't (currently) in use when rendering in Hugo, // but the pages needs to be assigned an output format, @@ -370,7 +368,7 @@ func TestCreateSiteOutputFormatsInvalidConfig(t *testing.T) { assert := require.New(t) outputsConfig := map[string]interface{}{ - KindHome: []string{"FOO", "JSON"}, + page.KindHome: []string{"FOO", "JSON"}, } cfg := viper.New() @@ -384,7 +382,7 @@ func TestCreateSiteOutputFormatsEmptyConfig(t *testing.T) { assert := require.New(t) outputsConfig := map[string]interface{}{ - KindHome: []string{}, + page.KindHome: []string{}, } cfg := viper.New() @@ -392,14 +390,14 @@ func TestCreateSiteOutputFormatsEmptyConfig(t *testing.T) { outputs, err := createSiteOutputFormats(output.DefaultFormats, cfg) assert.NoError(err) - assert.Equal(output.Formats{output.HTMLFormat, output.RSSFormat}, outputs[KindHome]) + assert.Equal(output.Formats{output.HTMLFormat, output.RSSFormat}, outputs[page.KindHome]) } func TestCreateSiteOutputFormatsCustomFormats(t *testing.T) { assert := require.New(t) outputsConfig := map[string]interface{}{ - KindHome: []string{}, + page.KindHome: []string{}, } cfg := viper.New() @@ -412,5 +410,5 @@ func TestCreateSiteOutputFormatsCustomFormats(t *testing.T) { outputs, err := createSiteOutputFormats(output.Formats{customRSS, customHTML}, cfg) assert.NoError(err) - assert.Equal(output.Formats{customHTML, customRSS}, outputs[KindHome]) + assert.Equal(output.Formats{customHTML, customRSS}, outputs[page.KindHome]) } diff --git a/hugolib/site_render.go b/hugolib/site_render.go index 4ce2b4c53..a6cf4bafa 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,17 +19,44 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/output" "github.com/pkg/errors" - "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/page/pagemeta" ) +type siteRenderContext struct { + cfg *BuildCfg + + // Zero based index for all output formats combined. + sitesOutIdx int + + // Zero based index of the output formats configured within a Site. + outIdx int + + multihost bool +} + +// Whether to render 404.html, robotsTXT.txt which usually is rendered +// once only in the site root. +func (s siteRenderContext) renderSingletonPages() bool { + if s.multihost { + // 1 per site + return s.outIdx == 0 + } + + // 1 for all sites + return s.sitesOutIdx == 0 + +} + // renderPages renders pages each corresponding to a markdown file. // TODO(bep np doc -func (s *Site) renderPages(cfg *BuildCfg) error { +func (s *Site) renderPages(ctx *siteRenderContext) error { results := make(chan error) - pages := make(chan *Page) + pages := make(chan *pageState) errs := make(chan error) go s.errorCollator(results, errs) @@ -40,17 +67,25 @@ func (s *Site) renderPages(cfg *BuildCfg) error { for i := 0; i < numWorkers; i++ { wg.Add(1) - go pageRenderer(s, pages, results, wg) + go pageRenderer(ctx, s, pages, results, wg) } - if !cfg.PartialReRender && len(s.headlessPages) > 0 { + cfg := ctx.cfg + + if !cfg.PartialReRender && ctx.outIdx == 0 && len(s.headlessPages) > 0 { wg.Add(1) go headlessPagesPublisher(s, wg) } - for _, page := range s.Pages { +L: + for _, page := range s.workAllPages { if cfg.shouldRender(page) { - pages <- page + select { + case <-s.h.Done(): + break L + default: + pages <- page + } } } @@ -69,207 +104,99 @@ func (s *Site) renderPages(cfg *BuildCfg) error { func headlessPagesPublisher(s *Site, wg *sync.WaitGroup) { defer wg.Done() - for _, page := range s.headlessPages { - outFormat := page.outputFormats[0] // There is only one - if outFormat.Name != s.rc.Format.Name { - // Avoid double work. - continue - } - pageOutput, err := newPageOutput(page, false, false, outFormat) - if err == nil { - page.mainPageOutput = pageOutput - err = pageOutput.renderResources() - } - - if err != nil { - s.Log.ERROR.Printf("Failed to render resources for headless page %q: %s", page, err) + for _, p := range s.headlessPages { + if err := p.renderResources(); err != nil { + s.SendError(p.errorf(err, "failed to render page resources")) } } } -func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.WaitGroup) { - defer wg.Done() - - for page := range pages { - - for i, outFormat := range page.outputFormats { +func pageRenderer( + ctx *siteRenderContext, + s *Site, + pages <-chan *pageState, + results chan<- error, + wg *sync.WaitGroup) { - if outFormat.Name != page.s.rc.Format.Name { - // Will be rendered ... later. - continue - } + defer wg.Done() - var ( - pageOutput *PageOutput - err error - ) + for p := range pages { + f := p.outputFormat() - if i == 0 { - pageOutput = page.mainPageOutput - } else { - pageOutput, err = page.mainPageOutput.copyWithFormat(outFormat, true) - } + // TODO(bep) get rid of this odd construct. RSS is an output format. + if f.Name == "RSS" && !s.isEnabled(kindRSS) { + continue + } - if err != nil { - s.Log.ERROR.Printf("Failed to create output page for type %q for page %q: %s", outFormat.Name, page, err) + if ctx.outIdx == 0 { + if err := p.renderResources(); err != nil { + s.SendError(p.errorf(err, "failed to render page resources")) continue } + } - if pageOutput == nil { - panic("no pageOutput") - } - - // We only need to re-publish the resources if the output format is different - // from all of the previous (e.g. the "amp" use case). - shouldRender := i == 0 - if i > 0 { - for j := i; j >= 0; j-- { - if outFormat.Path != page.outputFormats[j].Path { - shouldRender = true - } else { - shouldRender = false - } - } - } - - if shouldRender { - if err := pageOutput.renderResources(); err != nil { - s.SendError(page.errorf(err, "failed to render page resources")) - continue - } - } - - var layouts []string - - if page.selfLayout != "" { - layouts = []string{page.selfLayout} - } else { - layouts, err = s.layouts(pageOutput) - if err != nil { - s.Log.ERROR.Printf("Failed to resolve layout for output %q for page %q: %s", outFormat.Name, page, err) - continue - } - } - - switch pageOutput.outputFormat.Name { + layouts, err := p.getLayouts() + if err != nil { + s.Log.ERROR.Printf("Failed to resolve layout for output %q for page %q: %s", f.Name, p, err) + continue + } - case "RSS": - if err := s.renderRSS(pageOutput); err != nil { - results <- err - } - default: - targetPath, err := pageOutput.targetPath() - if err != nil { - s.Log.ERROR.Printf("Failed to create target path for output %q for page %q: %s", outFormat.Name, page, err) - continue - } + targetPath := p.targetPaths().TargetFilename - s.Log.DEBUG.Printf("Render %s to %q with layouts %q", pageOutput.Kind, targetPath, layouts) + if targetPath == "" { + s.Log.ERROR.Printf("Failed to create target path for output %q for page %q: %s", f.Name, p, err) + continue + } - if err := s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "page "+pageOutput.FullFilePath(), targetPath, pageOutput, layouts...); err != nil { - results <- err - } + if err := s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "page "+p.Title(), targetPath, p, layouts...); err != nil { + results <- err + } - // Only render paginators for the main output format - if i == 0 && pageOutput.IsNode() { - if err := s.renderPaginator(pageOutput); err != nil { - results <- err - } - } + if p.paginator != nil && p.paginator.current != nil { + if err := s.renderPaginator(p, layouts); err != nil { + results <- err } - } } } // renderPaginator must be run after the owning Page has been rendered. -func (s *Site) renderPaginator(p *PageOutput) error { - if p.paginator != nil { - s.Log.DEBUG.Printf("Render paginator for page %q", p.Path()) - paginatePath := s.Cfg.GetString("paginatePath") - - // write alias for page 1 - addend := fmt.Sprintf("/%s/%d", paginatePath, 1) - target, err := p.createTargetPath(p.outputFormat, false, addend) - if err != nil { - return err - } +func (s *Site) renderPaginator(p *pageState, layouts []string) error { - // TODO(bep) do better - link := newOutputFormat(p.Page, p.outputFormat).Permalink() - if err := s.writeDestAlias(target, link, p.outputFormat, nil); err != nil { - return err - } + paginatePath := s.Cfg.GetString("paginatePath") - pagers := p.paginator.Pagers() - - for i, pager := range pagers { - if i == 0 { - // already created - continue - } + d := p.targetPathDescriptor + f := p.s.rc.Format + d.Type = f - pagerNode, err := p.copy() - if err != nil { - return err - } + // Rewind + p.paginator.current = p.paginator.current.First() - pagerNode.origOnCopy = p.Page + // Write alias for page 1 + d.Addends = fmt.Sprintf("/%s/%d", paginatePath, 1) + targetPaths := page.CreateTargetPaths(d) - pagerNode.paginator = pager - if pager.TotalPages() > 0 { - first, _ := pager.page(0) - pagerNode.Date = first.Date - pagerNode.Lastmod = first.Lastmod - } - - pageNumber := i + 1 - addend := fmt.Sprintf("/%s/%d", paginatePath, pageNumber) - targetPath, _ := p.targetPath(addend) - layouts, err := p.layouts() - - if err != nil { - return err - } - - if err := s.renderAndWritePage( - &s.PathSpec.ProcessingStats.PaginatorPages, - pagerNode.title, - targetPath, pagerNode, layouts...); err != nil { - return err - } - - } + if err := s.writeDestAlias(targetPaths.TargetFilename, p.Permalink(), f, nil); err != nil { + return err } - return nil -} -func (s *Site) renderRSS(p *PageOutput) error { + // Render pages for the rest + for current := p.paginator.current.Next(); current != nil; current = current.Next() { - if !s.isEnabled(kindRSS) { - return nil - } - - limit := s.Cfg.GetInt("rssLimit") - if limit >= 0 && len(p.Pages) > limit { - p.Pages = p.Pages[:limit] - p.data["Pages"] = p.Pages - } + p.paginator.current = current + d.Addends = fmt.Sprintf("/%s/%d", paginatePath, current.PageNumber()) + targetPaths := page.CreateTargetPaths(d) - layouts, err := s.layoutHandler.For( - p.layoutDescriptor, - p.outputFormat) - if err != nil { - return err - } + if err := s.renderAndWritePage( + &s.PathSpec.ProcessingStats.PaginatorPages, + p.Title(), + targetPaths.TargetFilename, p, layouts...); err != nil { + return err + } - targetPath, err := p.targetPath() - if err != nil { - return err } - return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Pages, p.title, - targetPath, p, layouts...) + return nil } func (s *Site) render404() error { @@ -277,33 +204,29 @@ func (s *Site) render404() error { return nil } - p := s.newNodePage(kind404) + p, err := newPageStandalone(&pageMeta{ + s: s, + kind: kind404, + urlPaths: pagemeta.URLPath{ + URL: path.Join(s.GetURLLanguageBasePath(), "404.html"), + }, + }, + output.HTMLFormat, + ) - p.title = "404 Page not found" - p.data["Pages"] = s.Pages - p.Pages = s.Pages - p.URLPath.URL = "404.html" - - if err := p.initTargetPathDescriptor(); err != nil { + if err != nil { return err } nfLayouts := []string{"404.html"} - htmlOut := output.HTMLFormat - htmlOut.BaseName = "404" - - pageOutput, err := newPageOutput(p, false, false, htmlOut) - if err != nil { - return err - } + targetPath := p.targetPaths().TargetFilename - targetPath, err := pageOutput.targetPath() - if err != nil { - s.Log.ERROR.Printf("Failed to create target path for page %q: %s", p, err) + if targetPath == "" { + return errors.New("failed to create targetPath for 404 page") } - return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "404 page", targetPath, pageOutput, nfLayouts...) + return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "404 page", targetPath, p, nfLayouts...) } func (s *Site) renderSitemap() error { @@ -311,50 +234,28 @@ func (s *Site) renderSitemap() error { return nil } - sitemapDefault := parseSitemap(s.Cfg.GetStringMap("sitemap")) - - n := s.newNodePage(kindSitemap) - - // Include all pages (regular, home page, taxonomies etc.) - pages := s.Pages - - page := s.newNodePage(kindSitemap) - page.URLPath.URL = "" - if err := page.initTargetPathDescriptor(); err != nil { - return err - } - page.Sitemap.ChangeFreq = sitemapDefault.ChangeFreq - page.Sitemap.Priority = sitemapDefault.Priority - page.Sitemap.Filename = sitemapDefault.Filename - - n.data["Pages"] = pages - n.Pages = pages + p, err := newPageStandalone(&pageMeta{ + s: s, + kind: kindSitemap, + urlPaths: pagemeta.URLPath{ + URL: s.siteCfg.sitemap.Filename, + }}, + output.HTMLFormat, + ) - // TODO(bep) we have several of these - if err := page.initTargetPathDescriptor(); err != nil { + if err != nil { return err } - // TODO(bep) this should be done somewhere else - for _, page := range pages { - if page.Sitemap.ChangeFreq == "" { - page.Sitemap.ChangeFreq = sitemapDefault.ChangeFreq - } - - if page.Sitemap.Priority == -1 { - page.Sitemap.Priority = sitemapDefault.Priority - } + targetPath := p.targetPaths().TargetFilename - if page.Sitemap.Filename == "" { - page.Sitemap.Filename = sitemapDefault.Filename - } + if targetPath == "" { + return errors.New("failed to create targetPath for sitemap") } smLayouts := []string{"sitemap.xml", "_default/sitemap.xml", "_internal/_default/sitemap.xml"} - addLanguagePrefix := n.Site.IsMultiLingual() - return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemap", - n.addLangPathPrefixIfFlagSet(page.Sitemap.Filename, addLanguagePrefix), n, smLayouts...) + return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemap", targetPath, p, smLayouts...) } func (s *Site) renderRobotsTXT() error { @@ -366,53 +267,50 @@ func (s *Site) renderRobotsTXT() error { return nil } - p := s.newNodePage(kindRobotsTXT) - if err := p.initTargetPathDescriptor(); err != nil { - return err - } - p.data["Pages"] = s.Pages - p.Pages = s.Pages - - rLayouts := []string{"robots.txt", "_default/robots.txt", "_internal/_default/robots.txt"} + p, err := newPageStandalone(&pageMeta{ + s: s, + kind: kindRobotsTXT, + urlPaths: pagemeta.URLPath{ + URL: path.Join(s.GetURLLanguageBasePath(), "robots.txt"), + }, + }, + output.RobotsTxtFormat) - pageOutput, err := newPageOutput(p, false, false, output.RobotsTxtFormat) if err != nil { return err } - targetPath, err := pageOutput.targetPath() - if err != nil { - s.Log.ERROR.Printf("Failed to create target path for page %q: %s", p, err) - } + rLayouts := []string{"robots.txt", "_default/robots.txt", "_internal/_default/robots.txt"} - return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "Robots Txt", targetPath, pageOutput, rLayouts...) + return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "Robots Txt", p.targetPaths().TargetFilename, p, rLayouts...) } // renderAliases renders shell pages that simply have a redirect in the header. func (s *Site) renderAliases() error { - for _, p := range s.Pages { - if len(p.Aliases) == 0 { + for _, p := range s.workAllPages { + + if len(p.Aliases()) == 0 { continue } - for _, f := range p.outputFormats { - if !f.IsHTML { + for _, of := range p.OutputFormats() { + if !of.Format.IsHTML { continue } - o := newOutputFormat(p, f) - plink := o.Permalink() + plink := of.Permalink() + f := of.Format - for _, a := range p.Aliases { + for _, a := range p.Aliases() { if f.Path != "" { // Make sure AMP and similar doesn't clash with regular aliases. a = path.Join(a, f.Path) } - lang := p.Lang() + lang := p.Language().Lang - if s.owner.multihost && !strings.HasPrefix(a, "/"+lang) { + if s.h.multihost && !strings.HasPrefix(a, "/"+lang) { // These need to be in its language root. a = path.Join(lang, a) } @@ -424,22 +322,32 @@ func (s *Site) renderAliases() error { } } - if s.owner.multilingual.enabled() && !s.owner.IsMultihost() { - html, found := s.outputFormatsConfig.GetByName("HTML") - if found { - mainLang := s.owner.multilingual.DefaultLang - if s.Info.defaultContentLanguageInSubdir { - mainLangURL := s.PathSpec.AbsURL(mainLang.Lang, false) - s.Log.DEBUG.Printf("Write redirect to main language %s: %s", mainLang, mainLangURL) - if err := s.publishDestAlias(true, "/", mainLangURL, html, nil); err != nil { - return err - } - } else { - mainLangURL := s.PathSpec.AbsURL("", false) - s.Log.DEBUG.Printf("Write redirect to main language %s: %s", mainLang, mainLangURL) - if err := s.publishDestAlias(true, mainLang.Lang, mainLangURL, html, nil); err != nil { - return err - } + return nil +} + +// renderMainLanguageRedirect creates a redirect to the main language home, +// depending on if it lives in sub folder (e.g. /en) or not. +func (s *Site) renderMainLanguageRedirect() error { + + if !s.h.multilingual.enabled() || s.h.IsMultihost() { + // No need for a redirect + return nil + } + + html, found := s.outputFormatsConfig.GetByName("HTML") + if found { + mainLang := s.h.multilingual.DefaultLang + if s.Info.defaultContentLanguageInSubdir { + mainLangURL := s.PathSpec.AbsURL(mainLang.Lang, false) + s.Log.DEBUG.Printf("Write redirect to main language %s: %s", mainLang, mainLangURL) + if err := s.publishDestAlias(true, "/", mainLangURL, html, nil); err != nil { + return err + } + } else { + mainLangURL := s.PathSpec.AbsURL("", false) + s.Log.DEBUG.Printf("Write redirect to main language %s: %s", mainLang, mainLangURL) + if err := s.publishDestAlias(true, mainLang.Lang, mainLangURL, html, nil); err != nil { + return err } } } diff --git a/hugolib/site_sections.go b/hugolib/site_sections.go index 38f6a3b6f..d383e6389 100644 --- a/hugolib/site_sections.go +++ b/hugolib/site_sections.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,18 +14,18 @@ package hugolib import ( - "fmt" "path" "strconv" "strings" - "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource" radix "github.com/hashicorp/go-immutable-radix" ) // Sections returns the top level sections. -func (s *SiteInfo) Sections() Pages { +func (s *SiteInfo) Sections() page.Pages { home, err := s.Home() if err == nil { return home.Sections() @@ -34,157 +34,23 @@ func (s *SiteInfo) Sections() Pages { } // Home is a shortcut to the home page, equivalent to .Site.GetPage "home". -func (s *SiteInfo) Home() (*Page, error) { - return s.GetPage(KindHome) +func (s *SiteInfo) Home() (page.Page, error) { + return s.s.home, nil } -// Parent returns a section's parent section or a page's section. -// To get a section's subsections, see Page's Sections method. -func (p *Page) Parent() *Page { - return p.parent -} - -// CurrentSection returns the page's current section or the page itself if home or a section. -// Note that this will return nil for pages that is not regular, home or section pages. -func (p *Page) CurrentSection() *Page { - v := p - if v.origOnCopy != nil { - v = v.origOnCopy - } - if v.IsHome() || v.IsSection() { - return v - } - - return v.parent -} - -// FirstSection returns the section on level 1 below home, e.g. "/docs". -// For the home page, this will return itself. -func (p *Page) FirstSection() *Page { - v := p - if v.origOnCopy != nil { - v = v.origOnCopy - } - - if v.parent == nil || v.parent.IsHome() { - return v - } - - parent := v.parent - for { - current := parent - parent = parent.parent - if parent == nil || parent.IsHome() { - return current - } - } - -} - -// InSection returns whether the given page is in the current section. -// Note that this will always return false for pages that are -// not either regular, home or section pages. -func (p *Page) InSection(other interface{}) (bool, error) { - if p == nil || other == nil { - return false, nil - } - - pp, err := unwrapPage(other) - if err != nil { - return false, err - } - - if pp == nil { - return false, nil - } - - return pp.CurrentSection() == p.CurrentSection(), nil -} - -// IsDescendant returns whether the current page is a descendant of the given page. -// Note that this method is not relevant for taxonomy lists and taxonomy terms pages. -func (p *Page) IsDescendant(other interface{}) (bool, error) { - if p == nil { - return false, nil - } - pp, err := unwrapPage(other) - if err != nil || pp == nil { - return false, err - } - - if pp.Kind == KindPage && len(p.sections) == len(pp.sections) { - // A regular page is never its section's descendant. - return false, nil - } - return helpers.HasStringsPrefix(p.sections, pp.sections), nil -} - -// IsAncestor returns whether the current page is an ancestor of the given page. -// Note that this method is not relevant for taxonomy lists and taxonomy terms pages. -func (p *Page) IsAncestor(other interface{}) (bool, error) { - if p == nil { - return false, nil - } +func (s *Site) assembleSections() pageStatePages { + var newPages pageStatePages - pp, err := unwrapPage(other) - if err != nil || pp == nil { - return false, err - } - - if p.Kind == KindPage && len(p.sections) == len(pp.sections) { - // A regular page is never its section's ancestor. - return false, nil - } - - return helpers.HasStringsPrefix(pp.sections, p.sections), nil -} - -// Eq returns whether the current page equals the given page. -// Note that this is more accurate than doing `{{ if eq $page $otherPage }}` -// since a Page can be embedded in another type. -func (p *Page) Eq(other interface{}) bool { - pp, err := unwrapPage(other) - if err != nil { - return false - } - - return p == pp -} - -func unwrapPage(in interface{}) (*Page, error) { - switch v := in.(type) { - case *Page: - return v, nil - case *PageOutput: - return v.Page, nil - case *PageWithoutContent: - return v.Page, nil - case nil: - return nil, nil - default: - return nil, fmt.Errorf("%T not supported", in) - } -} - -// Sections returns this section's subsections, if any. -// Note that for non-sections, this method will always return an empty list. -func (p *Page) Sections() Pages { - return p.subSections -} - -func (s *Site) assembleSections() Pages { - var newPages Pages - - if !s.isEnabled(KindSection) { + if !s.isEnabled(page.KindSection) { return newPages } // Maps section kind pages to their path, i.e. "my/section" - sectionPages := make(map[string]*Page) + sectionPages := make(map[string]*pageState) // The sections with content files will already have been created. - for _, sect := range s.findPagesByKind(KindSection) { - sectionPages[path.Join(sect.sections...)] = sect + for _, sect := range s.findWorkPagesByKind(page.KindSection) { + sectionPages[sect.SectionsPath()] = sect } const ( @@ -196,39 +62,44 @@ func (s *Site) assembleSections() Pages { var ( inPages = radix.New().Txn() inSections = radix.New().Txn() - undecided Pages + undecided pageStatePages ) - home := s.findFirstPageByKindIn(KindHome, s.Pages) + home := s.findFirstWorkPageByKindIn(page.KindHome) - for i, p := range s.Pages { - if p.Kind != KindPage { + for i, p := range s.workAllPages { + + if p.Kind() != page.KindPage { continue } - if len(p.sections) == 0 { + sections := p.SectionsEntries() + + if len(sections) == 0 { // Root level pages. These will have the home page as their Parent. p.parent = home continue } - sectionKey := path.Join(p.sections...) - sect, found := sectionPages[sectionKey] + sectionKey := p.SectionsPath() + _, found := sectionPages[sectionKey] + + if !found && len(sections) == 1 { - if !found && len(p.sections) == 1 { // We only create content-file-less sections for the root sections. - sect = s.newSectionPage(p.sections[0]) - sectionPages[sectionKey] = sect - newPages = append(newPages, sect) + n := s.newPage(page.KindSection, sections[0]) + + sectionPages[sectionKey] = n + newPages = append(newPages, n) found = true } - if len(p.sections) > 1 { + if len(sections) > 1 { // Create the root section if not found. - _, rootFound := sectionPages[p.sections[0]] + _, rootFound := sectionPages[sections[0]] if !rootFound { - sect = s.newSectionPage(p.sections[0]) - sectionPages[p.sections[0]] = sect + sect := s.newPage(page.KindSection, sections[0]) + sectionPages[sections[0]] = sect newPages = append(newPages, sect) } } @@ -246,13 +117,14 @@ func (s *Site) assembleSections() Pages { // given a content file in /content/a/b/c/_index.md, we cannot create just // the c section. for _, sect := range sectionPages { - for i := len(sect.sections); i > 0; i-- { - sectionPath := sect.sections[:i] + sections := sect.SectionsEntries() + for i := len(sections); i > 0; i-- { + sectionPath := sections[:i] sectionKey := path.Join(sectionPath...) - sect, found := sectionPages[sectionKey] + _, found := sectionPages[sectionKey] if !found { - sect = s.newSectionPage(sectionPath[len(sectionPath)-1]) - sect.sections = sectionPath + sect = s.newPage(page.KindSection, sectionPath[len(sectionPath)-1]) + sect.m.sections = sectionPath sectionPages[sectionKey] = sect newPages = append(newPages, sect) } @@ -265,33 +137,36 @@ func (s *Site) assembleSections() Pages { } var ( - currentSection *Page - children Pages + currentSection *pageState + children page.Pages + dates *resource.Dates rootSections = inSections.Commit().Root() ) for i, p := range undecided { // Now we can decide where to put this page into the tree. - sectionKey := path.Join(p.sections...) + sectionKey := p.SectionsPath() + _, v, _ := rootSections.LongestPrefix([]byte(sectionKey)) - sect := v.(*Page) - pagePath := path.Join(path.Join(sect.sections...), sectSectKey, "u", strconv.Itoa(i)) + sect := v.(*pageState) + pagePath := path.Join(path.Join(sect.SectionsEntries()...), sectSectKey, "u", strconv.Itoa(i)) inPages.Insert([]byte(pagePath), p) } var rootPages = inPages.Commit().Root() rootPages.Walk(func(path []byte, v interface{}) bool { - p := v.(*Page) + p := v.(*pageState) - if p.Kind == KindSection { + if p.Kind() == page.KindSection { if currentSection != nil { // A new section - currentSection.setPagePages(children) + currentSection.setPages(children) } currentSection = p - children = make(Pages, 0) + children = make(page.Pages, 0) + dates = &resource.Dates{} return false @@ -300,27 +175,31 @@ func (s *Site) assembleSections() Pages { // Regular page p.parent = currentSection children = append(children, p) + dates.UpdateDateAndLastmodIfAfter(p) return false }) if currentSection != nil { - currentSection.setPagePages(children) + currentSection.setPages(children) + currentSection.m.Dates = *dates + } // Build the sections hierarchy for _, sect := range sectionPages { - if len(sect.sections) == 1 { - sect.parent = home + sections := sect.SectionsEntries() + if len(sections) == 1 { + if home != nil { + sect.parent = home + } } else { - parentSearchKey := path.Join(sect.sections[:len(sect.sections)-1]...) + parentSearchKey := path.Join(sect.SectionsEntries()[:len(sections)-1]...) _, v, _ := rootSections.LongestPrefix([]byte(parentSearchKey)) - p := v.(*Page) + p := v.(*pageState) sect.parent = p } - if sect.parent != nil { - sect.parent.subSections = append(sect.parent.subSections, sect) - } + sect.addSectionToParent() } var ( @@ -331,24 +210,13 @@ func (s *Site) assembleSections() Pages { maxSectionWeight int ) - mainSections, mainSectionsFound = s.Info.Params[sectionsParamIdLower] + mainSections, mainSectionsFound = s.Info.Params()[sectionsParamIdLower] for _, sect := range sectionPages { - if sect.parent != nil { - sect.parent.subSections.sort() - } - - for i, p := range sect.Pages { - if i > 0 { - p.NextInSection = sect.Pages[i-1] - } - if i < len(sect.Pages)-1 { - p.PrevInSection = sect.Pages[i+1] - } - } + sect.sortParentSections() if !mainSectionsFound { - weight := len(sect.Pages) + (len(sect.Sections()) * 5) + weight := len(sect.Pages()) + (len(sect.Sections()) * 5) if weight >= maxSectionWeight { mainSections = []string{sect.Section()} maxSectionWeight = weight @@ -357,16 +225,9 @@ func (s *Site) assembleSections() Pages { } // Try to make this as backwards compatible as possible. - s.Info.Params[sectionsParamId] = mainSections - s.Info.Params[sectionsParamIdLower] = mainSections + s.Info.Params()[sectionsParamId] = mainSections + s.Info.Params()[sectionsParamIdLower] = mainSections return newPages } - -func (p *Page) setPagePages(pages Pages) { - pages.sort() - p.Pages = pages - p.data = make(map[string]interface{}) - p.data["Pages"] = pages -} diff --git a/hugolib/site_sections_test.go b/hugolib/site_sections_test.go index 1987d2bcb..3adfb2b57 100644 --- a/hugolib/site_sections_test.go +++ b/hugolib/site_sections_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import ( "testing" "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/resources/page" "github.com/stretchr/testify/require" ) @@ -117,65 +118,66 @@ PAG|{{ .Title }}|{{ $sect.InSection . }} s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - require.Len(t, s.RegularPages, 21) + require.Len(t, s.RegularPages(), 21) tests := []struct { sections string - verify func(p *Page) + verify func(assert *require.Assertions, p page.Page) }{ - {"elsewhere", func(p *Page) { - assert.Len(p.Pages, 1) - for _, p := range p.Pages { - assert.Equal([]string{"elsewhere"}, p.sections) + {"elsewhere", func(assert *require.Assertions, p page.Page) { + assert.Len(p.Pages(), 1) + for _, p := range p.Pages() { + assert.Equal("elsewhere", p.SectionsPath()) } }}, - {"post", func(p *Page) { - assert.Len(p.Pages, 2) - for _, p := range p.Pages { + {"post", func(assert *require.Assertions, p page.Page) { + assert.Len(p.Pages(), 2) + for _, p := range p.Pages() { assert.Equal("post", p.Section()) } }}, - {"empty1", func(p *Page) { + {"empty1", func(assert *require.Assertions, p page.Page) { // > b,c - assert.NotNil(p.s.getPage(KindSection, "empty1", "b")) - assert.NotNil(p.s.getPage(KindSection, "empty1", "b", "c")) + assert.NotNil(getPage(p, "/empty1/b")) + assert.NotNil(getPage(p, "/empty1/b/c")) }}, - {"empty2", func(p *Page) { + {"empty2", func(assert *require.Assertions, p page.Page) { // > b,c,d where b and d have content files. - b := p.s.getPage(KindSection, "empty2", "b") + b := getPage(p, "/empty2/b") assert.NotNil(b) - assert.Equal("T40_-1", b.title) - c := p.s.getPage(KindSection, "empty2", "b", "c") + assert.Equal("T40_-1", b.Title()) + c := getPage(p, "/empty2/b/c") + assert.NotNil(c) - assert.Equal("Cs", c.title) - d := p.s.getPage(KindSection, "empty2", "b", "c", "d") + assert.Equal("Cs", c.Title()) + d := getPage(p, "/empty2/b/c/d") + assert.NotNil(d) - assert.Equal("T41_-1", d.title) + assert.Equal("T41_-1", d.Title()) assert.False(c.Eq(d)) assert.True(c.Eq(c)) assert.False(c.Eq("asdf")) }}, - {"empty3", func(p *Page) { + {"empty3", func(assert *require.Assertions, p page.Page) { // b,c,d with regular page in b - b := p.s.getPage(KindSection, "empty3", "b") + b := getPage(p, "/empty3/b") assert.NotNil(b) - assert.Len(b.Pages, 1) - assert.Equal("empty3.md", b.Pages[0].File.LogicalName()) + assert.Len(b.Pages(), 1) + assert.Equal("empty3.md", b.Pages()[0].File().LogicalName()) }}, - {"empty3", func(p *Page) { - xxx := p.s.getPage(KindPage, "empty3", "nil") + {"empty3", func(assert *require.Assertions, p page.Page) { + xxx := getPage(p, "/empty3/nil") assert.Nil(xxx) - assert.Equal(xxx.Eq(nil), true) }}, - {"top", func(p *Page) { - assert.Equal("Tops", p.title) - assert.Len(p.Pages, 2) - assert.Equal("mypage2.md", p.Pages[0].LogicalName()) - assert.Equal("mypage3.md", p.Pages[1].LogicalName()) + {"top", func(assert *require.Assertions, p page.Page) { + assert.Equal("Tops", p.Title()) + assert.Len(p.Pages(), 2) + assert.Equal("mypage2.md", p.Pages()[0].File().LogicalName()) + assert.Equal("mypage3.md", p.Pages()[1].File().LogicalName()) home := p.Parent() assert.True(home.IsHome()) assert.Len(p.Sections(), 0) @@ -185,30 +187,31 @@ PAG|{{ .Title }}|{{ $sect.InSection . }} assert.True(active) assert.Equal(p, p.FirstSection()) }}, - {"l1", func(p *Page) { - assert.Equal("L1s", p.title) - assert.Len(p.Pages, 2) + {"l1", func(assert *require.Assertions, p page.Page) { + assert.Equal("L1s", p.Title()) + assert.Len(p.Pages(), 2) assert.True(p.Parent().IsHome()) assert.Len(p.Sections(), 2) }}, - {"l1,l2", func(p *Page) { - assert.Equal("T2_-1", p.title) - assert.Len(p.Pages, 3) - assert.Equal(p, p.Pages[0].Parent()) - assert.Equal("L1s", p.Parent().title) - assert.Equal("/l1/l2/", p.URLPath.URL) + {"l1,l2", func(assert *require.Assertions, p page.Page) { + assert.Equal("T2_-1", p.Title()) + assert.Len(p.Pages(), 3) + assert.Equal(p, p.Pages()[0].Parent()) + assert.Equal("L1s", p.Parent().Title()) assert.Equal("/l1/l2/", p.RelPermalink()) assert.Len(p.Sections(), 1) - for _, child := range p.Pages { + for _, child := range p.Pages() { + assert.Equal(p, child.CurrentSection()) active, err := child.InSection(p) assert.NoError(err) + assert.True(active) active, err = p.InSection(child) assert.NoError(err) assert.True(active) - active, err = p.InSection(p.s.getPage(KindHome)) + active, err = p.InSection(getPage(p, "/")) assert.NoError(err) assert.False(active) @@ -227,25 +230,25 @@ PAG|{{ .Title }}|{{ $sect.InSection . }} assert.True(isDescendant) } - assert.Equal(p, p.CurrentSection()) + assert.True(p.Eq(p.CurrentSection())) }}, - {"l1,l2_2", func(p *Page) { - assert.Equal("T22_-1", p.title) - assert.Len(p.Pages, 2) - assert.Equal(filepath.FromSlash("l1/l2_2/page_2_2_1.md"), p.Pages[0].Path()) - assert.Equal("L1s", p.Parent().title) + {"l1,l2_2", func(assert *require.Assertions, p page.Page) { + assert.Equal("T22_-1", p.Title()) + assert.Len(p.Pages(), 2) + assert.Equal(filepath.FromSlash("l1/l2_2/page_2_2_1.md"), p.Pages()[0].File().Path()) + assert.Equal("L1s", p.Parent().Title()) assert.Len(p.Sections(), 0) }}, - {"l1,l2,l3", func(p *Page) { - var nilp *Page + {"l1,l2,l3", func(assert *require.Assertions, p page.Page) { + nilp, _ := p.GetPage("this/does/not/exist") - assert.Equal("T3_-1", p.title) - assert.Len(p.Pages, 2) - assert.Equal("T2_-1", p.Parent().title) + assert.Equal("T3_-1", p.Title()) + assert.Len(p.Pages(), 2) + assert.Equal("T2_-1", p.Parent().Title()) assert.Len(p.Sections(), 0) - l1 := p.s.getPage(KindSection, "l1") + l1 := getPage(p, "/l1") isDescendant, err := l1.IsDescendant(p) assert.NoError(err) assert.False(isDescendant) @@ -274,32 +277,35 @@ PAG|{{ .Title }}|{{ $sect.InSection . }} assert.False(isAncestor) }}, - {"perm a,link", func(p *Page) { - assert.Equal("T9_-1", p.title) + {"perm a,link", func(assert *require.Assertions, p page.Page) { + assert.Equal("T9_-1", p.Title()) assert.Equal("/perm-a/link/", p.RelPermalink()) - assert.Len(p.Pages, 4) - first := p.Pages[0] + assert.Len(p.Pages(), 4) + first := p.Pages()[0] assert.Equal("/perm-a/link/t1_1/", first.RelPermalink()) th.assertFileContent("public/perm-a/link/t1_1/index.html", "Single|T1_1") - last := p.Pages[3] + last := p.Pages()[3] assert.Equal("/perm-a/link/t1_5/", last.RelPermalink()) }}, } - home := s.getPage(KindHome) + home := s.getPage(page.KindHome) for _, test := range tests { - sections := strings.Split(test.sections, ",") - p := s.getPage(KindSection, sections...) - assert.NotNil(p, fmt.Sprint(sections)) - - if p.Pages != nil { - assert.Equal(p.Pages, p.data["Pages"]) - } - assert.NotNil(p.Parent(), fmt.Sprintf("Parent nil: %q", test.sections)) - test.verify(p) + t.Run(fmt.Sprintf("sections %s", test.sections), func(t *testing.T) { + assert := require.New(t) + sections := strings.Split(test.sections, ",") + p := s.getPage(page.KindSection, sections...) + assert.NotNil(p, fmt.Sprint(sections)) + + if p.Pages() != nil { + assert.Equal(p.Pages(), p.Data().(page.Data).Pages()) + } + assert.NotNil(p.Parent(), fmt.Sprintf("Parent nil: %q", test.sections)) + test.verify(assert, p) + }) } assert.NotNil(home) @@ -307,7 +313,7 @@ PAG|{{ .Title }}|{{ $sect.InSection . }} assert.Len(home.Sections(), 9) assert.Equal(home.Sections(), s.Info.Sections()) - rootPage := s.getPage(KindPage, "mypage.md") + rootPage := s.getPage(page.KindPage, "mypage.md") assert.NotNil(rootPage) assert.True(rootPage.Parent().IsHome()) @@ -317,7 +323,7 @@ PAG|{{ .Title }}|{{ $sect.InSection . }} // If we later decide to do something about this, we will have to do some normalization in // getPage. // TODO(bep) - sectionWithSpace := s.getPage(KindSection, "Spaces in Section") + sectionWithSpace := s.getPage(page.KindSection, "Spaces in Section") require.NotNil(t, sectionWithSpace) require.Equal(t, "/spaces-in-section/", sectionWithSpace.RelPermalink()) diff --git a/hugolib/site_test.go b/hugolib/site_test.go index bf46c313a..98fe1ff4f 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ package hugolib import ( "fmt" + "os" "path/filepath" "strings" "testing" @@ -24,6 +25,7 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/resources/page" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -77,13 +79,13 @@ func TestDraftAndFutureRender(t *testing.T) { // Testing Defaults.. Only draft:true and publishDate in the past should be rendered s := siteSetup(t) - if len(s.RegularPages) != 1 { + if len(s.RegularPages()) != 1 { t.Fatal("Draft or Future dated content published unexpectedly") } // only publishDate in the past should be rendered s = siteSetup(t, "buildDrafts", true) - if len(s.RegularPages) != 2 { + if len(s.RegularPages()) != 2 { t.Fatal("Future Dated Posts published unexpectedly") } @@ -92,7 +94,7 @@ func TestDraftAndFutureRender(t *testing.T) { "buildDrafts", false, "buildFuture", true) - if len(s.RegularPages) != 2 { + if len(s.RegularPages()) != 2 { t.Fatal("Draft posts published unexpectedly") } @@ -101,7 +103,7 @@ func TestDraftAndFutureRender(t *testing.T) { "buildDrafts", true, "buildFuture", true) - if len(s.RegularPages) != 4 { + if len(s.RegularPages()) != 4 { t.Fatal("Drafts or Future posts not included as expected") } @@ -128,17 +130,17 @@ func TestFutureExpirationRender(t *testing.T) { s := siteSetup(t) - if len(s.AllPages) != 1 { - if len(s.RegularPages) > 1 { + if len(s.AllPages()) != 1 { + if len(s.RegularPages()) > 1 { t.Fatal("Expired content published unexpectedly") } - if len(s.RegularPages) < 1 { + if len(s.RegularPages()) < 1 { t.Fatal("Valid content expired unexpectedly") } } - if s.AllPages[0].title == "doc2" { + if s.AllPages()[0].Title() == "doc2" { t.Fatal("Expired content published unexpectedly") } } @@ -156,8 +158,8 @@ func TestLastChange(t *testing.T) { s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - require.False(t, s.Info.LastChange.IsZero(), "Site.LastChange is zero") - require.Equal(t, 2017, s.Info.LastChange.Year(), "Site.LastChange should be set to the page with latest Lastmod (year 2017)") + require.False(t, s.Info.LastChange().IsZero(), "Site.LastChange is zero") + require.Equal(t, 2017, s.Info.LastChange().Year(), "Site.LastChange should be set to the page with latest Lastmod (year 2017)") } // Issue #_index @@ -170,7 +172,7 @@ func TestPageWithUnderScoreIndexInFilename(t *testing.T) { s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - require.Len(t, s.RegularPages, 1) + require.Len(t, s.RegularPages(), 1) } @@ -255,7 +257,7 @@ THE END.`, refShortcode), WithTemplate: createWithTemplateFromNameValues("_default/single.html", "{{.Content}}")}, BuildCfg{}) - require.Len(t, s.RegularPages, 4) + require.Len(t, s.RegularPages(), 4) th := testHelper{s.Cfg, s.Fs, t} @@ -328,13 +330,13 @@ func doTestShouldAlwaysHaveUglyURLs(t *testing.T, uglyURLs bool) { {filepath.FromSlash("public/index.html"), "Home Sweet Home."}, {filepath.FromSlash(expectedPagePath), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"}, {filepath.FromSlash("public/404.html"), "Page Not Found."}, - {filepath.FromSlash("public/index.xml"), "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>\n<root>RSS</root>"}, - {filepath.FromSlash("public/sitemap.xml"), "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>\n<root>SITEMAP</root>"}, + {filepath.FromSlash("public/index.xml"), "<root>RSS</root>"}, + {filepath.FromSlash("public/sitemap.xml"), "<root>SITEMAP</root>"}, // Issue #1923 {filepath.FromSlash("public/ugly.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>doc2 <em>content</em></p>\n"}, } - for _, p := range s.RegularPages { + for _, p := range s.RegularPages() { assert.False(t, p.IsHome()) } @@ -406,7 +408,7 @@ func doTestSectionNaming(t *testing.T, canonify, uglify, pluralize bool) { } writeSource(t, fs, filepath.Join("layouts", "_default/single.html"), "{{.Content}}") - writeSource(t, fs, filepath.Join("layouts", "_default/list.html"), "{{.Title}}") + writeSource(t, fs, filepath.Join("layouts", "_default/list.html"), "{{ .Kind }}|{{.Title}}") s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) @@ -491,6 +493,7 @@ func TestSkipRender(t *testing.T) { for _, test := range tests { file, err := fs.Destination.Open(test.doc) if err != nil { + helpers.PrintFs(fs.Destination, "public", os.Stdout) t.Fatalf("Did not find %s in target.", test.doc) } @@ -610,40 +613,40 @@ func TestOrderedPages(t *testing.T) { s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - if s.getPage(KindSection, "sect").Pages[1].title != "Three" || s.getPage(KindSection, "sect").Pages[2].title != "Four" { + if s.getPage(page.KindSection, "sect").Pages()[1].Title() != "Three" || s.getPage(page.KindSection, "sect").Pages()[2].Title() != "Four" { t.Error("Pages in unexpected order.") } - bydate := s.RegularPages.ByDate() + bydate := s.RegularPages().ByDate() - if bydate[0].title != "One" { - t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "One", bydate[0].title) + if bydate[0].Title() != "One" { + t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "One", bydate[0].Title()) } rev := bydate.Reverse() - if rev[0].title != "Three" { - t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "Three", rev[0].title) + if rev[0].Title() != "Three" { + t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "Three", rev[0].Title()) } - bypubdate := s.RegularPages.ByPublishDate() + bypubdate := s.RegularPages().ByPublishDate() - if bypubdate[0].title != "One" { - t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "One", bypubdate[0].title) + if bypubdate[0].Title() != "One" { + t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "One", bypubdate[0].Title()) } rbypubdate := bypubdate.Reverse() - if rbypubdate[0].title != "Three" { - t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "Three", rbypubdate[0].title) + if rbypubdate[0].Title() != "Three" { + t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "Three", rbypubdate[0].Title()) } - bylength := s.RegularPages.ByLength() - if bylength[0].title != "One" { - t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "One", bylength[0].title) + bylength := s.RegularPages().ByLength() + if bylength[0].Title() != "One" { + t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "One", bylength[0].Title()) } rbylength := bylength.Reverse() - if rbylength[0].title != "Four" { - t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "Four", rbylength[0].title) + if rbylength[0].Title() != "Four" { + t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "Four", rbylength[0].Title()) } } @@ -668,7 +671,7 @@ func TestGroupedPages(t *testing.T) { writeSourcesToSource(t, "content", fs, groupedSources...) s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - rbysection, err := s.RegularPages.GroupBy("Section", "desc") + rbysection, err := s.RegularPages().GroupBy("Section", "desc") if err != nil { t.Fatalf("Unable to make PageGroup array: %s", err) } @@ -682,14 +685,14 @@ func TestGroupedPages(t *testing.T) { if rbysection[2].Key != "sect1" { t.Errorf("PageGroup array in unexpected order. Third group key should be '%s', got '%s'", "sect1", rbysection[2].Key) } - if rbysection[0].Pages[0].title != "Four" { - t.Errorf("PageGroup has an unexpected page. First group's pages should have '%s', got '%s'", "Four", rbysection[0].Pages[0].title) + if rbysection[0].Pages[0].Title() != "Four" { + t.Errorf("PageGroup has an unexpected page. First group's pages should have '%s', got '%s'", "Four", rbysection[0].Pages[0].Title()) } if len(rbysection[2].Pages) != 2 { t.Errorf("PageGroup has unexpected number of pages. Third group should have '%d' pages, got '%d' pages", 2, len(rbysection[2].Pages)) } - bytype, err := s.RegularPages.GroupBy("Type", "asc") + bytype, err := s.RegularPages().GroupBy("Type", "asc") if err != nil { t.Fatalf("Unable to make PageGroup array: %s", err) } @@ -702,14 +705,14 @@ func TestGroupedPages(t *testing.T) { if bytype[2].Key != "sect3" { t.Errorf("PageGroup array in unexpected order. Third group key should be '%s', got '%s'", "sect3", bytype[2].Key) } - if bytype[2].Pages[0].title != "Four" { - t.Errorf("PageGroup has an unexpected page. Third group's data should have '%s', got '%s'", "Four", bytype[0].Pages[0].title) + if bytype[2].Pages[0].Title() != "Four" { + t.Errorf("PageGroup has an unexpected page. Third group's data should have '%s', got '%s'", "Four", bytype[0].Pages[0].Title()) } if len(bytype[0].Pages) != 2 { t.Errorf("PageGroup has unexpected number of pages. First group should have '%d' pages, got '%d' pages", 2, len(bytype[2].Pages)) } - bydate, err := s.RegularPages.GroupByDate("2006-01", "asc") + bydate, err := s.RegularPages().GroupByDate("2006-01", "asc") if err != nil { t.Fatalf("Unable to make PageGroup array: %s", err) } @@ -720,7 +723,7 @@ func TestGroupedPages(t *testing.T) { t.Errorf("PageGroup array in unexpected order. Second group key should be '%s', got '%s'", "2012-01", bydate[1].Key) } - bypubdate, err := s.RegularPages.GroupByPublishDate("2006") + bypubdate, err := s.RegularPages().GroupByPublishDate("2006") if err != nil { t.Fatalf("Unable to make PageGroup array: %s", err) } @@ -730,14 +733,14 @@ func TestGroupedPages(t *testing.T) { if bypubdate[1].Key != "0001" { t.Errorf("PageGroup array in unexpected order. Second group key should be '%s', got '%s'", "0001", bypubdate[1].Key) } - if bypubdate[0].Pages[0].title != "Three" { - t.Errorf("PageGroup has an unexpected page. Third group's pages should have '%s', got '%s'", "Three", bypubdate[0].Pages[0].title) + if bypubdate[0].Pages[0].Title() != "Three" { + t.Errorf("PageGroup has an unexpected page. Third group's pages should have '%s', got '%s'", "Three", bypubdate[0].Pages[0].Title()) } if len(bypubdate[0].Pages) != 3 { t.Errorf("PageGroup has unexpected number of pages. First group should have '%d' pages, got '%d' pages", 3, len(bypubdate[0].Pages)) } - byparam, err := s.RegularPages.GroupByParam("my_param", "desc") + byparam, err := s.RegularPages().GroupByParam("my_param", "desc") if err != nil { t.Fatalf("Unable to make PageGroup array: %s", err) } @@ -750,19 +753,19 @@ func TestGroupedPages(t *testing.T) { if byparam[2].Key != "bar" { t.Errorf("PageGroup array in unexpected order. Third group key should be '%s', got '%s'", "bar", byparam[2].Key) } - if byparam[2].Pages[0].title != "Three" { - t.Errorf("PageGroup has an unexpected page. Third group's pages should have '%s', got '%s'", "Three", byparam[2].Pages[0].title) + if byparam[2].Pages[0].Title() != "Three" { + t.Errorf("PageGroup has an unexpected page. Third group's pages should have '%s', got '%s'", "Three", byparam[2].Pages[0].Title()) } if len(byparam[0].Pages) != 2 { t.Errorf("PageGroup has unexpected number of pages. First group should have '%d' pages, got '%d' pages", 2, len(byparam[0].Pages)) } - _, err = s.RegularPages.GroupByParam("not_exist") + _, err = s.RegularPages().GroupByParam("not_exist") if err == nil { t.Errorf("GroupByParam didn't return an expected error") } - byOnlyOneParam, err := s.RegularPages.GroupByParam("only_one") + byOnlyOneParam, err := s.RegularPages().GroupByParam("only_one") if err != nil { t.Fatalf("Unable to make PageGroup array: %s", err) } @@ -773,7 +776,7 @@ func TestGroupedPages(t *testing.T) { t.Errorf("PageGroup array in unexpected order. First group key should be '%s', got '%s'", "yes", byOnlyOneParam[0].Key) } - byParamDate, err := s.RegularPages.GroupByParamDate("my_date", "2006-01") + byParamDate, err := s.RegularPages().GroupByParamDate("my_date", "2006-01") if err != nil { t.Fatalf("Unable to make PageGroup array: %s", err) } @@ -783,8 +786,8 @@ func TestGroupedPages(t *testing.T) { if byParamDate[1].Key != "1979-05" { t.Errorf("PageGroup array in unexpected order. Second group key should be '%s', got '%s'", "1979-05", byParamDate[1].Key) } - if byParamDate[1].Pages[0].title != "One" { - t.Errorf("PageGroup has an unexpected page. Second group's pages should have '%s', got '%s'", "One", byParamDate[1].Pages[0].title) + if byParamDate[1].Pages[0].Title() != "One" { + t.Errorf("PageGroup has an unexpected page. Second group's pages should have '%s', got '%s'", "One", byParamDate[1].Pages[0].Title()) } if len(byParamDate[0].Pages) != 2 { t.Errorf("PageGroup has unexpected number of pages. First group should have '%d' pages, got '%d' pages", 2, len(byParamDate[2].Pages)) @@ -840,16 +843,16 @@ func TestWeightedTaxonomies(t *testing.T) { writeSourcesToSource(t, "content", fs, sources...) s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - if s.Taxonomies["tags"]["a"][0].Page.title != "foo" { - t.Errorf("Pages in unexpected order, 'foo' expected first, got '%v'", s.Taxonomies["tags"]["a"][0].Page.title) + if s.Taxonomies["tags"]["a"][0].Page.Title() != "foo" { + t.Errorf("Pages in unexpected order, 'foo' expected first, got '%v'", s.Taxonomies["tags"]["a"][0].Page.Title()) } - if s.Taxonomies["categories"]["d"][0].Page.title != "bar" { - t.Errorf("Pages in unexpected order, 'bar' expected first, got '%v'", s.Taxonomies["categories"]["d"][0].Page.title) + if s.Taxonomies["categories"]["d"][0].Page.Title() != "bar" { + t.Errorf("Pages in unexpected order, 'bar' expected first, got '%v'", s.Taxonomies["categories"]["d"][0].Page.Title()) } - if s.Taxonomies["categories"]["e"][0].Page.title != "bza" { - t.Errorf("Pages in unexpected order, 'bza' expected first, got '%v'", s.Taxonomies["categories"]["e"][0].Page.title) + if s.Taxonomies["categories"]["e"][0].Page.Title() != "bza" { + t.Errorf("Pages in unexpected order, 'bza' expected first, got '%v'", s.Taxonomies["categories"]["e"][0].Page.Title()) } } @@ -897,7 +900,7 @@ func TestRefLinking(t *testing.T) { t.Parallel() site := setupLinkingMockSite(t) - currentPage := site.getPage(KindPage, "level2/level3/start.md") + currentPage := site.getPage(page.KindPage, "level2/level3/start.md") if currentPage == nil { t.Fatalf("failed to find current page in site") } @@ -952,8 +955,8 @@ func TestRefLinking(t *testing.T) { // TODO: and then the failure cases. } -func checkLinkCase(site *Site, link string, currentPage *Page, relative bool, outputFormat string, expected string, t *testing.T, i int) { +func checkLinkCase(site *Site, link string, currentPage page.Page, relative bool, outputFormat string, expected string, t *testing.T, i int) { if out, err := site.refLink(link, currentPage, relative, outputFormat); err != nil || out != expected { - t.Errorf("[%d] Expected %q from %q to resolve to %q, got %q - error: %s", i, link, currentPage.absoluteSourceRef(), expected, out, err) + t.Fatalf("[%d] Expected %q from %q to resolve to %q, got %q - error: %s", i, link, currentPage.Path(), expected, out, err) } } diff --git a/hugolib/site_url_test.go b/hugolib/site_url_test.go index 5b9d19e0d..10aa3bb28 100644 --- a/hugolib/site_url_test.go +++ b/hugolib/site_url_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/resources/page" + "html/template" "github.com/gohugoio/hugo/deps" @@ -115,14 +117,14 @@ Do not go gentle into that good night. s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{SkipRender: true}) - assert.Len(s.RegularPages, 2) + assert.Len(s.RegularPages(), 2) - notUgly := s.getPage(KindPage, "sect1/p1.md") + notUgly := s.getPage(page.KindPage, "sect1/p1.md") assert.NotNil(notUgly) assert.Equal("sect1", notUgly.Section()) assert.Equal("/sect1/p1/", notUgly.RelPermalink()) - ugly := s.getPage(KindPage, "sect2/p2.md") + ugly := s.getPage(page.KindPage, "sect2/p2.md") assert.NotNil(ugly) assert.Equal("sect2", ugly.Section()) assert.Equal("/sect2/p2.html", ugly.RelPermalink()) @@ -173,9 +175,9 @@ Do not go gentle into that good night. s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - assert.Len(s.RegularPages, 10) + assert.Len(s.RegularPages(), 10) - sect1 := s.getPage(KindSection, "sect1") + sect1 := s.getPage(page.KindSection, "sect1") assert.NotNil(sect1) assert.Equal("/ss1/", sect1.RelPermalink()) th.assertFileContent(filepath.Join("public", "ss1", "index.html"), "P1|URL: /ss1/|Next: /ss1/page/2/") diff --git a/hugolib/sitemap.go b/hugolib/sitemap.go deleted file mode 100644 index 64d6f5b7a..000000000 --- a/hugolib/sitemap.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "github.com/spf13/cast" - jww "github.com/spf13/jwalterweatherman" -) - -// Sitemap configures the sitemap to be generated. -type Sitemap struct { - ChangeFreq string - Priority float64 - Filename string -} - -func parseSitemap(input map[string]interface{}) Sitemap { - sitemap := Sitemap{Priority: -1, Filename: "sitemap.xml"} - - for key, value := range input { - switch key { - case "changefreq": - sitemap.ChangeFreq = cast.ToString(value) - case "priority": - sitemap.Priority = cast.ToFloat64(value) - case "filename": - sitemap.Filename = cast.ToString(value) - default: - jww.WARN.Printf("Unknown Sitemap field: %s\n", key) - } - } - - return sitemap -} diff --git a/hugolib/sitemap_test.go b/hugolib/sitemap_test.go index 002f772d8..cab13d356 100644 --- a/hugolib/sitemap_test.go +++ b/hugolib/sitemap_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,10 +18,10 @@ import ( "reflect" - "github.com/stretchr/testify/require" - + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/tpl" + "github.com/stretchr/testify/require" ) const sitemapTemplate = `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> @@ -86,14 +86,14 @@ func doTestSitemapOutput(t *testing.T, internal bool) { func TestParseSitemap(t *testing.T) { t.Parallel() - expected := Sitemap{Priority: 3.0, Filename: "doo.xml", ChangeFreq: "3"} + expected := config.Sitemap{Priority: 3.0, Filename: "doo.xml", ChangeFreq: "3"} input := map[string]interface{}{ "changefreq": "3", "priority": 3.0, "filename": "doo.xml", "unknown": "ignore", } - result := parseSitemap(input) + result := config.DecodeSitemap(config.Sitemap{}, input) if !reflect.DeepEqual(expected, result) { t.Errorf("Got \n%v expected \n%v", result, expected) diff --git a/hugolib/taxonomy.go b/hugolib/taxonomy.go index c8447d1ba..9d9e4f9ec 100644 --- a/hugolib/taxonomy.go +++ b/hugolib/taxonomy.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,11 @@ package hugolib import ( "fmt" + "path" "sort" + + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/resource" ) // The TaxonomyList is a list of all taxonomies and their values @@ -28,44 +32,30 @@ func (tl TaxonomyList) String() string { // A Taxonomy is a map of keywords to a list of pages. // For example -// TagTaxonomy['technology'] = WeightedPages -// TagTaxonomy['go'] = WeightedPages2 -type Taxonomy map[string]WeightedPages - -// WeightedPages is a list of Pages with their corresponding (and relative) weight -// [{Weight: 30, Page: *1}, {Weight: 40, Page: *2}] -type WeightedPages []WeightedPage - -// A WeightedPage is a Page with a weight. -type WeightedPage struct { - Weight int - *Page -} - -func (w WeightedPage) String() string { - return fmt.Sprintf("WeightedPage(%d,%q)", w.Weight, w.Page.title) -} +// TagTaxonomy['technology'] = page.WeightedPages +// TagTaxonomy['go'] = page.WeightedPages +type Taxonomy map[string]page.WeightedPages // OrderedTaxonomy is another representation of an Taxonomy using an array rather than a map. // Important because you can't order a map. type OrderedTaxonomy []OrderedTaxonomyEntry // OrderedTaxonomyEntry is similar to an element of a Taxonomy, but with the key embedded (as name) -// e.g: {Name: Technology, WeightedPages: Taxonomyedpages} +// e.g: {Name: Technology, page.WeightedPages: TaxonomyPages} type OrderedTaxonomyEntry struct { - Name string - WeightedPages WeightedPages + Name string + page.WeightedPages } // Get the weighted pages for the given key. -func (i Taxonomy) Get(key string) WeightedPages { +func (i Taxonomy) Get(key string) page.WeightedPages { return i[key] } // Count the weighted pages for the given key. func (i Taxonomy) Count(key string) int { return len(i[key]) } -func (i Taxonomy) add(key string, w WeightedPage) { +func (i Taxonomy) add(key string, w page.WeightedPage) { i[key] = append(i[key], w) } @@ -110,7 +100,7 @@ func (i Taxonomy) ByCount() OrderedTaxonomy { } // Pages returns the Pages for this taxonomy. -func (ie OrderedTaxonomyEntry) Pages() Pages { +func (ie OrderedTaxonomyEntry) Pages() page.Pages { return ie.WeightedPages.Pages() } @@ -165,60 +155,81 @@ func (s *orderedTaxonomySorter) Less(i, j int) bool { return s.by(&s.taxonomy[i], &s.taxonomy[j]) } -// Pages returns the Pages in this weighted page set. -func (wp WeightedPages) Pages() Pages { - pages := make(Pages, len(wp)) - for i := range wp { - pages[i] = wp[i].Page - } - return pages -} - -// Prev returns the previous Page relative to the given Page in -// this weighted page set. -func (wp WeightedPages) Prev(cur *Page) *Page { - for x, c := range wp { - if c.Page.UniqueID() == cur.UniqueID() { - if x == 0 { - return wp[len(wp)-1].Page - } - return wp[x-1].Page - } - } - return nil +// taxonomyNodeInfo stores additional metadata about a taxonomy. +type taxonomyNodeInfo struct { + plural string + + // Maps "tags" to "tag". + singular string + + // The term key as used in the taxonomy map, e.g "tag1". + // The value is normalized for paths, but may or not be lowercased + // depending on the disablePathToLower setting. + termKey string + + // The original, unedited term name. Useful for titles etc. + term string + + dates resource.Dates + + parent *taxonomyNodeInfo + + // Either of Kind taxonomyTerm (parent) or taxonomy + owner page.Page } -// Next returns the next Page relative to the given Page in -// this weighted page set. -func (wp WeightedPages) Next(cur *Page) *Page { - for x, c := range wp { - if c.Page.UniqueID() == cur.UniqueID() { - if x < len(wp)-1 { - return wp[x+1].Page - } - return wp[0].Page - } +func (t *taxonomyNodeInfo) UpdateFromPage(p page.Page) { + + // Select the latest dates + t.dates.UpdateDateAndLastmodIfAfter(p) +} + +func (t *taxonomyNodeInfo) TransferValues(p *pageState) { + t.owner = p + if p.Lastmod().IsZero() && p.Date().IsZero() { + p.m.Dates.UpdateDateAndLastmodIfAfter(t.dates) } - return nil } -func (wp WeightedPages) Len() int { return len(wp) } -func (wp WeightedPages) Swap(i, j int) { wp[i], wp[j] = wp[j], wp[i] } +// callback sent to the child nodes. +func (t *taxonomyNodeInfo) getOwner() page.Page { + return t.owner +} -// Sort stable sorts this weighted page set. -func (wp WeightedPages) Sort() { sort.Stable(wp) } +// Maps either plural or plural/term to a taxonomy node. +// TODO(bep) consolidate somehow with s.Taxonomies +type taxonomyNodeInfos map[string]*taxonomyNodeInfo -// Count returns the number of pages in this weighted page set. -func (wp WeightedPages) Count() int { return len(wp) } +func (t taxonomyNodeInfos) key(parts ...string) string { + return path.Join(parts...) +} -func (wp WeightedPages) Less(i, j int) bool { - if wp[i].Weight == wp[j].Weight { - if wp[i].Page.Date.Equal(wp[j].Page.Date) { - return wp[i].Page.title < wp[j].Page.title - } - return wp[i].Page.Date.After(wp[i].Page.Date) +func (t taxonomyNodeInfos) GetOrCreate(plural, termKey, term string) *taxonomyNodeInfo { + key := t.key(plural, termKey) + + n, found := t[key] + if found { + return n } - return wp[i].Weight < wp[j].Weight + + n = &taxonomyNodeInfo{ + plural: plural, + termKey: termKey, + term: term, + } + + t[key] = n + + return n } -// TODO mimic PagesSorter for WeightedPages +func (t taxonomyNodeInfos) Get(sections ...string) *taxonomyNodeInfo { + key := t.key(sections...) + + n, found := t[key] + if found { + return n + } + + return nil +} diff --git a/hugolib/taxonomy_test.go b/hugolib/taxonomy_test.go index 1ae9fae22..2501ed2e4 100644 --- a/hugolib/taxonomy_test.go +++ b/hugolib/taxonomy_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,6 +16,9 @@ package hugolib import ( "fmt" "path/filepath" + + "github.com/gohugoio/hugo/resources/page" + "reflect" "strings" "testing" @@ -25,7 +28,7 @@ import ( "github.com/gohugoio/hugo/deps" ) -func TestByCountOrderOfTaxonomies(t *testing.T) { +func TestTaxonomiesCountOrder(t *testing.T) { t.Parallel() taxonomies := make(map[string]string) @@ -36,37 +39,42 @@ func TestByCountOrderOfTaxonomies(t *testing.T) { cfg.Set("taxonomies", taxonomies) - writeSource(t, fs, filepath.Join("content", "page.md"), pageYamlWithTaxonomiesA) + const pageContent = `--- +tags: ['a', 'B', 'c'] +categories: 'd' +--- +YAML frontmatter with tags and categories taxonomy.` + + writeSource(t, fs, filepath.Join("content", "page.md"), pageContent) s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) st := make([]string, 0) for _, t := range s.Taxonomies["tags"].ByCount() { - st = append(st, t.Name) + st = append(st, t.Page().Title()+":"+t.Name) } - if !reflect.DeepEqual(st, []string{"a", "b", "c"}) { - t.Fatalf("ordered taxonomies do not match [a, b, c]. Got: %s", st) + expect := []string{"a:a", "B:b", "c:c"} + + if !reflect.DeepEqual(st, expect) { + t.Fatalf("ordered taxonomies mismatch, expected\n%v\ngot\n%q", expect, st) } } // func TestTaxonomiesWithAndWithoutContentFile(t *testing.T) { for _, uglyURLs := range []bool{false, true} { - for _, preserveTaxonomyNames := range []bool{false, true} { - t.Run(fmt.Sprintf("uglyURLs=%t,preserveTaxonomyNames=%t", uglyURLs, preserveTaxonomyNames), func(t *testing.T) { - doTestTaxonomiesWithAndWithoutContentFile(t, preserveTaxonomyNames, uglyURLs) - }) - } + t.Run(fmt.Sprintf("uglyURLs=%t", uglyURLs), func(t *testing.T) { + doTestTaxonomiesWithAndWithoutContentFile(t, uglyURLs) + }) } } -func doTestTaxonomiesWithAndWithoutContentFile(t *testing.T, preserveTaxonomyNames, uglyURLs bool) { +func doTestTaxonomiesWithAndWithoutContentFile(t *testing.T, uglyURLs bool) { t.Parallel() siteConfig := ` baseURL = "http://example.com/blog" -preserveTaxonomyNames = %t uglyURLs = %t paginate = 1 defaultContentLanguage = "en" @@ -94,23 +102,17 @@ permalinkeds: # Doc ` - siteConfig = fmt.Sprintf(siteConfig, preserveTaxonomyNames, uglyURLs) + siteConfig = fmt.Sprintf(siteConfig, uglyURLs) th, h := newTestSitesFromConfigWithDefaultTemplates(t, siteConfig) require.Len(t, h.Sites, 1) fs := th.Fs - if preserveTaxonomyNames { - writeSource(t, fs, "content/p1.md", fmt.Sprintf(pageTemplate, "t1/c1", "- tag1", "- cat1", "- o1", "- pl1")) - } else { - // Check lower-casing of tags - writeSource(t, fs, "content/p1.md", fmt.Sprintf(pageTemplate, "t1/c1", "- Tag1", "- cAt1", "- o1", "- pl1")) - - } - writeSource(t, fs, "content/p2.md", fmt.Sprintf(pageTemplate, "t2/c1", "- tag2", "- cat1", "- o1", "- pl1")) - writeSource(t, fs, "content/p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- pl1")) - writeSource(t, fs, "content/p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- pl1")) + writeSource(t, fs, "content/p1.md", fmt.Sprintf(pageTemplate, "t1/c1", "- Tag1", "- cAt1", "- o1", "- Pl1")) + writeSource(t, fs, "content/p2.md", fmt.Sprintf(pageTemplate, "t2/c1", "- tag2", "- cAt1", "- o1", "- Pl1")) + writeSource(t, fs, "content/p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- Pl1")) + writeSource(t, fs, "content/p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- Pl1")) writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10) writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/Tag1/_index.md", 10) @@ -133,45 +135,29 @@ permalinkeds: } // 1. - if preserveTaxonomyNames { - th.assertFileContent(pathFunc("public/categories/cat1/index.html"), "List", "cat1") - } else { - th.assertFileContent(pathFunc("public/categories/cat1/index.html"), "List", "Cat1") - } - + th.assertFileContent(pathFunc("public/categories/cat1/index.html"), "List", "cAt1") th.assertFileContent(pathFunc("public/categories/index.html"), "Terms List", "Category Terms") // 2. - if preserveTaxonomyNames { - th.assertFileContent(pathFunc("public/tags/tag2/index.html"), "List", "tag2") - } else { - th.assertFileContent(pathFunc("public/tags/tag2/index.html"), "List", "Tag2") - } + th.assertFileContent(pathFunc("public/tags/tag2/index.html"), "List", "tag2") th.assertFileContent(pathFunc("public/tags/tag1/index.html"), "List", "Tag1") th.assertFileContent(pathFunc("public/tags/index.html"), "Terms List", "Tags") // 3. - if preserveTaxonomyNames { - th.assertFileContent(pathFunc("public/others/o1/index.html"), "List", "o1") - } else { - th.assertFileContent(pathFunc("public/others/o1/index.html"), "List", "O1") - } + th.assertFileContent(pathFunc("public/others/o1/index.html"), "List", "o1") th.assertFileContent(pathFunc("public/others/index.html"), "Terms List", "Others") // 4. - if preserveTaxonomyNames { - th.assertFileContent(pathFunc("public/perma/pl1/index.html"), "List", "pl1") - } else { - th.assertFileContent(pathFunc("public/perma/pl1/index.html"), "List", "Pl1") - } + th.assertFileContent(pathFunc("public/perma/pl1/index.html"), "List", "Pl1") + // This looks kind of funky, but the taxonomy terms do not have a permalinks definition, // for good reasons. th.assertFileContent(pathFunc("public/permalinkeds/index.html"), "Terms List", "Permalinkeds") s := h.Sites[0] - // Make sure that each KindTaxonomyTerm page has an appropriate number - // of KindTaxonomy pages in its Pages slice. + // Make sure that each page.KindTaxonomyTerm page has an appropriate number + // of page.KindTaxonomy pages in its Pages slice. taxonomyTermPageCounts := map[string]int{ "tags": 2, "categories": 2, @@ -181,16 +167,16 @@ permalinkeds: } for taxonomy, count := range taxonomyTermPageCounts { - term := s.getPage(KindTaxonomyTerm, taxonomy) + term := s.getPage(page.KindTaxonomyTerm, taxonomy) require.NotNil(t, term) - require.Len(t, term.Pages, count) + require.Len(t, term.Pages(), count) - for _, page := range term.Pages { - require.Equal(t, KindTaxonomy, page.Kind) + for _, p := range term.Pages() { + require.Equal(t, page.KindTaxonomy, p.Kind()) } } - cat1 := s.getPage(KindTaxonomy, "categories", "cat1") + cat1 := s.getPage(page.KindTaxonomy, "categories", "cat1") require.NotNil(t, cat1) if uglyURLs { require.Equal(t, "/blog/categories/cat1.html", cat1.RelPermalink()) @@ -198,8 +184,8 @@ permalinkeds: require.Equal(t, "/blog/categories/cat1/", cat1.RelPermalink()) } - pl1 := s.getPage(KindTaxonomy, "permalinkeds", "pl1") - permalinkeds := s.getPage(KindTaxonomyTerm, "permalinkeds") + pl1 := s.getPage(page.KindTaxonomy, "permalinkeds", "pl1") + permalinkeds := s.getPage(page.KindTaxonomyTerm, "permalinkeds") require.NotNil(t, pl1) require.NotNil(t, permalinkeds) if uglyURLs { @@ -210,16 +196,9 @@ permalinkeds: require.Equal(t, "/blog/permalinkeds/", permalinkeds.RelPermalink()) } - // Issue #3070 preserveTaxonomyNames - if preserveTaxonomyNames { - helloWorld := s.getPage(KindTaxonomy, "others", "Hello Hugo world") - require.NotNil(t, helloWorld) - require.Equal(t, "Hello Hugo world", helloWorld.title) - } else { - helloWorld := s.getPage(KindTaxonomy, "others", "hello-hugo-world") - require.NotNil(t, helloWorld) - require.Equal(t, "Hello Hugo World", helloWorld.title) - } + helloWorld := s.getPage(page.KindTaxonomy, "others", "hello-hugo-world") + require.NotNil(t, helloWorld) + require.Equal(t, "Hello Hugo world", helloWorld.Title()) // Issue #2977 th.assertFileContent(pathFunc("public/empties/index.html"), "Terms List", "Empties") @@ -282,21 +261,65 @@ title: "This is S3s" s := b.H.Sites[0] - ta := s.findPagesByKind(KindTaxonomy) - te := s.findPagesByKind(KindTaxonomyTerm) + ta := s.findPagesByKind(page.KindTaxonomy) + te := s.findPagesByKind(page.KindTaxonomyTerm) assert.Equal(4, len(te)) assert.Equal(7, len(ta)) - b.AssertFileContent("public/news/categories/a/index.html", "Taxonomy List Page 1|A|Hello|https://example.com/news/categories/a/|") + b.AssertFileContent("public/news/categories/a/index.html", "Taxonomy List Page 1|a|Hello|https://example.com/news/categories/a/|") b.AssertFileContent("public/news/categories/b/index.html", "Taxonomy List Page 1|This is B|Hello|https://example.com/news/categories/b/|") - b.AssertFileContent("public/news/categories/d/e/index.html", "Taxonomy List Page 1|D/E|Hello|https://example.com/news/categories/d/e/|") + b.AssertFileContent("public/news/categories/d/e/index.html", "Taxonomy List Page 1|d/e|Hello|https://example.com/news/categories/d/e/|") b.AssertFileContent("public/news/categories/f/g/h/index.html", "Taxonomy List Page 1|This is H|Hello|https://example.com/news/categories/f/g/h/|") b.AssertFileContent("public/t1/t2/t3s/t4/t5/index.html", "Taxonomy List Page 1|This is T5|Hello|https://example.com/t1/t2/t3s/t4/t5/|") - b.AssertFileContent("public/t1/t2/t3s/t4/t5/t6/index.html", "Taxonomy List Page 1|T4/T5/T6|Hello|https://example.com/t1/t2/t3s/t4/t5/t6/|") + b.AssertFileContent("public/t1/t2/t3s/t4/t5/t6/index.html", "Taxonomy List Page 1|t4/t5/t6|Hello|https://example.com/t1/t2/t3s/t4/t5/t6/|") b.AssertFileContent("public/news/categories/index.html", "Taxonomy Term Page 1|News/Categories|Hello|https://example.com/news/categories/|") b.AssertFileContent("public/t1/t2/t3s/index.html", "Taxonomy Term Page 1|T1/T2/T3s|Hello|https://example.com/t1/t2/t3s/|") b.AssertFileContent("public/s1/s2/s3s/index.html", "Taxonomy Term Page 1|This is S3s|Hello|https://example.com/s1/s2/s3s/|") } + +// https://github.com/gohugoio/hugo/issues/5719 +func TestTaxonomiesNextGenLoops(t *testing.T) { + b := newTestSitesBuilder(t).WithSimpleConfigFile() + + b.WithTemplatesAdded("index.html", ` +<h1>Tags</h1> +<ul> + {{ range .Site.Taxonomies.tags }} + <li><a href="{{ .Page.Permalink }}">{{ .Page.Title }}</a> {{ .Count }}</li> + {{ end }} +</ul> + +`) + + b.WithTemplatesAdded("_default/terms.html", ` +<h1>Terms</h1> +<ul> + {{ range .Data.Terms.Alphabetical }} + <li><a href="{{ .Page.Permalink }}">{{ .Page.Title }}</a> {{ .Count }}</li> + {{ end }} +</ul> +`) + + for i := 0; i < 10; i++ { + b.WithContent(fmt.Sprintf("page%d.md", i+1), ` +--- +Title: "Taxonomy!" +tags: ["Hugo Rocks!", "Rocks I say!" ] +categories: ["This is Cool", "And new" ] +--- + +Content. + + `) + } + + b.CreateSites().Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", `<li><a href="http://example.com/tags/hugo-rocks/">Hugo Rocks!</a> 10</li>`) + b.AssertFileContent("public/categories/index.html", `<li><a href="http://example.com/categories/this-is-cool/">This is Cool</a> 10</li>`) + b.AssertFileContent("public/tags/index.html", `<li><a href="http://example.com/tags/rocks-i-say/">Rocks I say!</a> 10</li>`) + +} diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 64d1ff96a..7de2280c7 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -14,11 +14,11 @@ import ( "strings" "text/template" - "github.com/gohugoio/hugo/langs" - "github.com/sanity-io/litter" - + "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/resources/page" + "github.com/sanity-io/litter" "github.com/spf13/afero" "github.com/gohugoio/hugo/helpers" @@ -387,6 +387,7 @@ func (s *sitesBuilder) build(cfg BuildCfg, shouldFail bool) *sitesBuilder { } } if err != nil && !shouldFail { + herrors.PrintStackTrace(err) s.Fatalf("Build failed: %s", err) } else if err == nil && shouldFail { s.Fatalf("Expected error") @@ -418,10 +419,10 @@ date: "2018-02-28" "content/sect/doc1.nn.md", contentTemplate, } - listTemplateCommon = "{{ $p := .Paginator }}{{ $p.PageNumber }}|{{ .Title }}|{{ i18n \"hello\" }}|{{ .Permalink }}|Pager: {{ template \"_internal/pagination.html\" . }}" + listTemplateCommon = "{{ $p := .Paginator }}{{ $p.PageNumber }}|{{ .Title }}|{{ i18n \"hello\" }}|{{ .Permalink }}|Pager: {{ template \"_internal/pagination.html\" . }}|Kind: {{ .Kind }}|Content: {{ .Content }}" defaultTemplates = []string{ - "_default/single.html", "Single: {{ .Title }}|{{ i18n \"hello\" }}|{{.Lang}}|{{ .Content }}", + "_default/single.html", "Single: {{ .Title }}|{{ i18n \"hello\" }}|{{.Language.Lang}}|RelPermalink: {{ .RelPermalink }}|Permalink: {{ .Permalink }}|{{ .Content }}|Resources: {{ range .Resources }}{{ .MediaType }}: {{ .RelPermalink}} -- {{ end }}|Summary: {{ .Summary }}|Truncated: {{ .Truncated }}", "_default/list.html", "List Page " + listTemplateCommon, "index.html", "{{ $p := .Paginator }}Default Home Page {{ $p.PageNumber }}: {{ .Title }}|{{ .IsHome }}|{{ i18n \"hello\" }}|{{ .Permalink }}|{{ .Site.Data.hugo.slogan }}|String Resource: {{ ( \"Hugo Pipes\" | resources.FromString \"text/pipes.txt\").RelPermalink }}", "index.fr.html", "{{ $p := .Paginator }}French Home Page {{ $p.PageNumber }}: {{ .Title }}|{{ .IsHome }}|{{ i18n \"hello\" }}|{{ .Permalink }}|{{ .Site.Data.hugo.slogan }}|String Resource: {{ ( \"Hugo Pipes\" | resources.FromString \"text/pipes.txt\").RelPermalink }}", @@ -432,6 +433,9 @@ date: "2018-02-28" // A shortcode in multiple languages "shortcodes/lingo.html", "LingoDefault", "shortcodes/lingo.fr.html", "LingoFrench", + // Special templates + "404.html", "404|{{ .Lang }}|{{ .Title }}", + "robots.txt", "robots|{{ .Lang }}|{{ .Title }}", } defaultI18n = []string{ @@ -469,18 +473,25 @@ func (s *sitesBuilder) Fatalf(format string, args ...interface{}) { } func Fatalf(t testing.TB, format string, args ...interface{}) { - trace := trace() + trace := stackTrace() format = format + "\n%s" args = append(args, trace) t.Fatalf(format, args...) } -func trace() string { +func stackTrace() string { return strings.Join(assert.CallerInfo(), "\n\r\t\t\t") } +func (s *sitesBuilder) AssertFileContentFn(filename string, f func(s string) bool) { + content := s.FileContent(filename) + if !f(content) { + s.Fatalf("Assert failed for %q", filename) + } +} + func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { - content := readDestination(s.T, s.Fs, filename) + content := s.FileContent(filename) for _, match := range matches { if !strings.Contains(content, match) { s.Fatalf("No match for %q in content for %s\n%s\n%q", match, filename, content, content) @@ -488,6 +499,10 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { } } +func (s *sitesBuilder) FileContent(filename string) string { + return readDestination(s.T, s.Fs, filename) +} + func (s *sitesBuilder) AssertObject(expected string, object interface{}) { got := s.dumper.Sdump(object) expected = strings.TrimSpace(expected) @@ -502,7 +517,7 @@ func (s *sitesBuilder) AssertObject(expected string, object interface{}) { func (s *sitesBuilder) AssertFileContentRe(filename string, matches ...string) { content := readDestination(s.T, s.Fs, filename) for _, match := range matches { - r := regexp.MustCompile(match) + r := regexp.MustCompile("(?s)" + match) if !r.MatchString(content) { s.Fatalf("No match for %q in content for %s\n%q", match, filename, content) } @@ -555,32 +570,6 @@ func (th testHelper) replaceDefaultContentLanguageValue(value string) string { return value } -func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *helpers.PathSpec { - l := langs.NewDefaultLanguage(v) - ps, _ := helpers.NewPathSpec(fs, l) - return ps -} - -func newTestDefaultPathSpec(t *testing.T) *helpers.PathSpec { - v := viper.New() - // Easier to reason about in tests. - v.Set("disablePathToLower", true) - v.Set("contentDir", "content") - v.Set("dataDir", "data") - v.Set("i18nDir", "i18n") - v.Set("layoutDir", "layouts") - v.Set("archetypeDir", "archetypes") - v.Set("assetDir", "assets") - v.Set("resourceDir", "resources") - v.Set("publishDir", "public") - fs := hugofs.NewDefault(v) - ps, err := helpers.NewPathSpec(fs, v) - if err != nil { - t.Fatal(err) - } - return ps -} - func newTestCfg() (*viper.Viper, *hugofs.Fs) { v := viper.New() @@ -597,27 +586,6 @@ func newTestCfg() (*viper.Viper, *hugofs.Fs) { } -// newTestSite creates a new site in the English language with in-memory Fs. -// The site will have a template system loaded and ready to use. -// Note: This is only used in single site tests. -func newTestSite(t testing.TB, configKeyValues ...interface{}) *Site { - - cfg, fs := newTestCfg() - - for i := 0; i < len(configKeyValues); i += 2 { - cfg.Set(configKeyValues[i].(string), configKeyValues[i+1]) - } - - d := deps.DepsCfg{Fs: fs, Cfg: cfg} - - s, err := NewSiteForCfg(d) - - if err != nil { - Fatalf(t, "Failed to create Site: %s", err) - } - return s -} - func newTestSitesFromConfig(t testing.TB, afs afero.Fs, tomlConfig string, layoutPathContentPairs ...string) (testHelper, *HugoSites) { if len(layoutPathContentPairs)%2 != 0 { Fatalf(t, "Layouts must be provided in pairs") @@ -696,11 +664,28 @@ func writeSourcesToSource(t *testing.T, base string, fs *hugofs.Fs, sources ...[ } } -func dumpPages(pages ...*Page) { +func getPage(in page.Page, ref string) page.Page { + p, err := in.GetPage(ref) + if err != nil { + panic(err) + } + return p +} + +func dumpPages(pages ...page.Page) { + fmt.Println("---------") for i, p := range pages { - fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s Len Sections(): %d\n", + fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s\n", i+1, - p.Kind, p.title, p.RelPermalink(), p.Path(), p.sections, len(p.Sections())) + p.Kind(), p.Title(), p.RelPermalink(), p.Path(), p.SectionsPath()) + } +} + +func dumpSPages(pages ...*pageState) { + for i, p := range pages { + fmt.Printf("%d: Kind: %s Title: %-10s RelPermalink: %-10s Path: %-10s sections: %s\n", + i+1, + p.Kind(), p.Title(), p.RelPermalink(), p.Path(), p.SectionsPath()) } } @@ -722,8 +707,8 @@ func printStringIndexes(s string) { fmt.Println() } - } + func isCI() bool { return os.Getenv("CI") != "" } @@ -731,3 +716,21 @@ func isCI() bool { func isGo111() bool { return strings.Contains(runtime.Version(), "1.11") } + +// See https://github.com/golang/go/issues/19280 +// Not in use. +var parallelEnabled = true + +func parallel(t *testing.T) { + if parallelEnabled { + t.Parallel() + } +} + +// Useful to debug nilpointers/panics in templates. +// Put "defer recoverStack()" in top of the failing function. +func recoverStack() { + if r := recover(); r != nil { + fmt.Println(printStackTrace(1000)) + } +} diff --git a/hugolib/translations.go b/hugolib/translations.go index 2682363f0..072ce33e5 100644 --- a/hugolib/translations.go +++ b/hugolib/translations.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,47 +13,41 @@ package hugolib -// Translations represent the other translations for a given page. The -// string here is the language code, as affected by the `post.LANG.md` -// filename. -type Translations map[string]*Page +import ( + "github.com/gohugoio/hugo/resources/page" +) -func pagesToTranslationsMap(pages []*Page) map[string]Translations { - out := make(map[string]Translations) +func pagesToTranslationsMap(sites []*Site) map[string]page.Pages { + out := make(map[string]page.Pages) - for _, page := range pages { - base := page.TranslationKey() + for _, s := range sites { + for _, p := range s.workAllPages { + // TranslationKey is implemented for all page types. + base := p.TranslationKey() - pageTranslation, present := out[base] - if !present { - pageTranslation = make(Translations) - } + pageTranslations, found := out[base] + if !found { + pageTranslations = make(page.Pages, 0) + } - pageLang := page.Lang() - if pageLang == "" { - continue + pageTranslations = append(pageTranslations, p) + out[base] = pageTranslations } - - pageTranslation[pageLang] = page - out[base] = pageTranslation } return out } -func assignTranslationsToPages(allTranslations map[string]Translations, pages []*Page) { - for _, page := range pages { - page.translations = page.translations[:0] - base := page.TranslationKey() - trans, exist := allTranslations[base] - if !exist { - continue - } +func assignTranslationsToPages(allTranslations map[string]page.Pages, sites []*Site) { + for _, s := range sites { + for _, p := range s.workAllPages { + base := p.TranslationKey() + translations, found := allTranslations[base] + if !found { + continue + } - for _, translatedPage := range trans { - page.translations = append(page.translations, translatedPage) + p.setTranslations(translations) } - - pageBy(languagePageSort).Sort(page.translations) } } |