diff options
author | Bjørn Erik Pedersen <[email protected]> | 2024-03-17 11:12:33 +0100 |
---|---|---|
committer | Bjørn Erik Pedersen <[email protected]> | 2024-05-14 13:12:08 +0200 |
commit | e2d66e3218e180bbfca06ca3a29ce01957c513e9 (patch) | |
tree | ed29bb99cf16b75b6334e2fc618d31e80203e5d5 /hugolib | |
parent | 55dea41c1ab703f13b841389c6888815a033cf86 (diff) | |
download | hugo-e2d66e3218e180bbfca06ca3a29ce01957c513e9.tar.gz hugo-e2d66e3218e180bbfca06ca3a29ce01957c513e9.zip |
Create pages from _content.gotmpl
Closes #12427
Closes #12485
Closes #6310
Closes #5074
Diffstat (limited to 'hugolib')
-rw-r--r-- | hugolib/config_test.go | 2 | ||||
-rw-r--r-- | hugolib/content_map.go | 197 | ||||
-rw-r--r-- | hugolib/content_map_page.go | 184 | ||||
-rw-r--r-- | hugolib/doctree/nodeshiftree_test.go | 14 | ||||
-rw-r--r-- | hugolib/doctree/nodeshifttree.go | 55 | ||||
-rw-r--r-- | hugolib/doctree/simpletree.go | 17 | ||||
-rw-r--r-- | hugolib/doctree/support.go | 6 | ||||
-rw-r--r-- | hugolib/doctree/treeshifttree.go | 57 | ||||
-rw-r--r-- | hugolib/hugo_sites.go | 19 | ||||
-rw-r--r-- | hugolib/hugo_sites_build.go | 138 | ||||
-rw-r--r-- | hugolib/hugo_smoke_test.go | 2 | ||||
-rw-r--r-- | hugolib/page.go | 14 | ||||
-rw-r--r-- | hugolib/page__content.go | 45 | ||||
-rw-r--r-- | hugolib/page__meta.go | 76 | ||||
-rw-r--r-- | hugolib/page__new.go | 69 | ||||
-rw-r--r-- | hugolib/page__per_output.go | 12 | ||||
-rw-r--r-- | hugolib/pages_capture.go | 49 | ||||
-rw-r--r-- | hugolib/pagesfromdata/pagesfromgotmpl.go | 331 | ||||
-rw-r--r-- | hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go | 479 | ||||
-rw-r--r-- | hugolib/pagesfromdata/pagesfromgotmpl_test.go | 32 | ||||
-rw-r--r-- | hugolib/shortcode.go | 4 | ||||
-rw-r--r-- | hugolib/site.go | 29 | ||||
-rw-r--r-- | hugolib/site_new.go | 24 | ||||
-rw-r--r-- | hugolib/site_sections.go | 2 |
24 files changed, 1598 insertions, 259 deletions
diff --git a/hugolib/config_test.go b/hugolib/config_test.go index 8acf8b36e..aaba534f5 100644 --- a/hugolib/config_test.go +++ b/hugolib/config_test.go @@ -1144,7 +1144,7 @@ Home. enConfig := b.H.Sites[0].conf m, _ := enConfig.MediaTypes.Config.GetByType("text/html") - b.Assert(m.Suffixes(), qt.DeepEquals, []string{"html"}) + b.Assert(m.Suffixes(), qt.DeepEquals, []string{"html", "htm"}) svConfig := b.H.Sites[1].conf f, _ := svConfig.OutputFormats.Config.GetByName("html") diff --git a/hugolib/content_map.go b/hugolib/content_map.go index 62cabec51..b75ccdbcb 100644 --- a/hugolib/content_map.go +++ b/hugolib/content_map.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "path" "path/filepath" @@ -23,10 +24,13 @@ import ( "github.com/bep/logg" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/hugolib/pagesfromdata" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/page/pagemeta" "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/hugofs" @@ -51,9 +55,11 @@ type contentMapConfig struct { var _ contentNodeI = (*resourceSource)(nil) type resourceSource struct { - path *paths.Path - opener hugio.OpenReadSeekCloser - fi hugofs.FileMetaInfo + langIndex int + path *paths.Path + opener hugio.OpenReadSeekCloser + fi hugofs.FileMetaInfo + rc *pagemeta.ResourceConfig r resource.Resource } @@ -64,11 +70,7 @@ func (r resourceSource) clone() *resourceSource { } func (r *resourceSource) LangIndex() int { - if r.r != nil && r.isPage() { - return r.r.(*pageState).s.languagei - } - - return r.fi.Meta().LangIndex + return r.langIndex } func (r *resourceSource) MarkStale() { @@ -162,12 +164,13 @@ func (cfg contentMapConfig) getTaxonomyConfig(s string) (v viewName) { return } -func (m *pageMap) AddFi(fi hugofs.FileMetaInfo) error { +func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCount uint64, resourceCount uint64, addErr error) { if fi.IsDir() { - return nil + return } insertResource := func(fim hugofs.FileMetaInfo) error { + resourceCount++ pi := fi.Meta().PathInfo key := pi.Base() tree := m.treeResources @@ -199,9 +202,9 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo) error { } key = pi.Base() - rs = &resourceSource{r: pageResource} + rs = &resourceSource{r: pageResource, langIndex: pageResource.s.languagei} } else { - rs = &resourceSource{path: pi, opener: r, fi: fim} + rs = &resourceSource{path: pi, opener: r, fi: fim, langIndex: fim.Meta().LangIndex} } tree.InsertIntoValuesDimension(key, rs) @@ -220,14 +223,27 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo) error { }, )) if err := insertResource(fi); err != nil { - return err + addErr = err + return } + case paths.PathTypeContentData: + pc, rc, err := m.addPagesFromGoTmplFi(fi, buildConfig) + pageCount += pc + resourceCount += rc + if err != nil { + addErr = err + return + } + default: m.s.Log.Trace(logg.StringFunc( func() string { return fmt.Sprintf("insert bundle: %q", fi.Meta().Filename) }, )) + + pageCount++ + // A content file. p, pi, err := m.s.h.newPage( &pageMeta{ @@ -237,17 +253,164 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo) error { }, ) if err != nil { - return err + addErr = err + return } if p == nil { // Disabled page. - return nil + return } - m.treePages.InsertWithLock(pi.Base(), p) + m.treePages.InsertIntoValuesDimensionWithLock(pi.Base(), p) } - return nil + return +} + +func (m *pageMap) addPagesFromGoTmplFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageCount uint64, resourceCount uint64, addErr error) { + meta := fi.Meta() + pi := meta.PathInfo + + m.s.Log.Trace(logg.StringFunc( + func() string { + return fmt.Sprintf("insert pages from data file: %q", fi.Meta().Filename) + }, + )) + + if !files.IsGoTmplExt(pi.Ext()) { + addErr = fmt.Errorf("unsupported data file extension %q", pi.Ext()) + return + } + + s := m.s.h.resolveSite(fi.Meta().Lang) + f := source.NewFileInfo(fi) + h := s.h + + // Make sure the layouts are initialized. + if _, err := h.init.layouts.Do(context.Background()); err != nil { + addErr = err + return + } + + contentAdapter := s.pageMap.treePagesFromTemplateAdapters.Get(pi.Base()) + var rebuild bool + if contentAdapter != nil { + // Rebuild + contentAdapter = contentAdapter.CloneForGoTmpl(fi) + rebuild = true + } else { + contentAdapter = pagesfromdata.NewPagesFromTemplate( + pagesfromdata.PagesFromTemplateOptions{ + GoTmplFi: fi, + Site: s, + DepsFromSite: func(s page.Site) pagesfromdata.PagesFromTemplateDeps { + ss := s.(*Site) + return pagesfromdata.PagesFromTemplateDeps{ + TmplFinder: ss.TextTmpl(), + TmplExec: ss.Tmpl(), + } + }, + DependencyManager: s.Conf.NewIdentityManager("pagesfromdata"), + Watching: s.Conf.Watching(), + HandlePage: func(pt *pagesfromdata.PagesFromTemplate, pc *pagemeta.PageConfig) error { + s := pt.Site.(*Site) + if err := pc.Compile(pt.GoTmplFi.Meta().PathInfo.Base(), true, "", s.Log, s.conf.MediaTypes.Config); err != nil { + return err + } + + ps, pi, err := h.newPage( + &pageMeta{ + f: f, + s: s, + pageMetaParams: &pageMetaParams{ + pageConfig: pc, + }, + }, + ) + if err != nil { + return err + } + + if ps == nil { + // Disabled page. + return nil + } + + u, n, replaced := s.pageMap.treePages.InsertIntoValuesDimensionWithLock(pi.Base(), ps) + + if h.isRebuild() { + if replaced { + pt.AddChange(n.GetIdentity()) + } else { + pt.AddChange(u.GetIdentity()) + } + } + + return nil + }, + HandleResource: func(pt *pagesfromdata.PagesFromTemplate, rc *pagemeta.ResourceConfig) error { + s := pt.Site.(*Site) + if err := rc.Compile( + pt.GoTmplFi.Meta().PathInfo.Base(), + s.Conf.PathParser(), + s.conf.MediaTypes.Config, + ); err != nil { + return err + } + + rs := &resourceSource{path: rc.PathInfo, rc: rc, opener: nil, fi: nil, langIndex: s.languagei} + + _, n, replaced := s.pageMap.treeResources.InsertIntoValuesDimensionWithLock(rc.PathInfo.Base(), rs) + + if h.isRebuild() && replaced { + pt.AddChange(n.GetIdentity()) + } + return nil + }, + }, + ) + + s.pageMap.treePagesFromTemplateAdapters.Insert(pi.Base(), contentAdapter) + + } + + handleBuildInfo := func(s *Site, bi pagesfromdata.BuildInfo) { + resourceCount += bi.NumResourcesAdded + pageCount += bi.NumPagesAdded + s.handleContentAdapterChanges(bi, buildConfig) + } + + bi, err := contentAdapter.Execute(context.Background()) + if err != nil { + addErr = err + return + } + handleBuildInfo(s, bi) + + if !rebuild && bi.EnableAllLanguages { + // Clone and insert the adapter for the other sites. + for _, ss := range s.h.Sites { + if s == ss { + continue + } + + clone := contentAdapter.CloneForSite(ss) + + // Make sure it gets executed for the first time. + bi, err := clone.Execute(context.Background()) + if err != nil { + addErr = err + return + } + handleBuildInfo(ss, bi) + + // Insert into the correct language tree so it get rebuilt on changes. + ss.pageMap.treePagesFromTemplateAdapters.Insert(pi.Base(), clone) + + } + } + + return } // The home page is represented with the zero string. diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index a0bff7472..5758cb6f6 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -34,7 +34,9 @@ import ( "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/hugofs/files" "github.com/gohugoio/hugo/hugolib/doctree" + "github.com/gohugoio/hugo/hugolib/pagesfromdata" "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/resources" "github.com/spf13/cast" @@ -100,6 +102,8 @@ type pageMap struct { cacheContentPlain *dynacache.Partition[string, *resources.StaleValue[contentPlainPlainWords]] contentTableOfContents *dynacache.Partition[string, *resources.StaleValue[contentTableOfContents]] + contentDataFileSeenItems *maps.Cache[string, map[uint64]bool] + cfg contentMapConfig } @@ -122,6 +126,10 @@ type pageTrees struct { // This tree contains all taxonomy entries, e.g "/tags/blue/page1" treeTaxonomyEntries *doctree.TreeShiftTree[*weightedContentNode] + // Stores the state for _content.gotmpl files. + // Mostly releveant for rebuilds. + treePagesFromTemplateAdapters *doctree.TreeShiftTree[*pagesfromdata.PagesFromTemplate] + // A slice of the resource trees. resourceTrees doctree.MutableTrees } @@ -222,6 +230,7 @@ func (t pageTrees) Shape(d, v int) *pageTrees { t.treePages = t.treePages.Shape(d, v) t.treeResources = t.treeResources.Shape(d, v) t.treeTaxonomyEntries = t.treeTaxonomyEntries.Shape(d, v) + t.treePagesFromTemplateAdapters = t.treePagesFromTemplateAdapters.Shape(d, v) t.createMutableTrees() return &t @@ -587,9 +596,9 @@ func (m *pageMap) getOrCreateResourcesForPage(ps *pageState) resource.Resources sort.SliceStable(res, lessFunc) - if len(ps.m.pageConfig.Resources) > 0 { + if len(ps.m.pageConfig.ResourcesMeta) > 0 { for i, r := range res { - res[i] = resources.CloneWithMetadataIfNeeded(ps.m.pageConfig.Resources, r) + res[i] = resources.CloneWithMetadataFromMapIfNeeded(ps.m.pageConfig.ResourcesMeta, r) } sort.SliceStable(res, lessFunc) } @@ -667,12 +676,13 @@ type contentNodeShifter struct { numLanguages int } -func (s *contentNodeShifter) Delete(n contentNodeI, dimension doctree.Dimension) (bool, bool) { +func (s *contentNodeShifter) Delete(n contentNodeI, dimension doctree.Dimension) (contentNodeI, bool, bool) { lidx := dimension[0] switch v := n.(type) { case contentNodeIs: - resource.MarkStale(v[lidx]) - wasDeleted := v[lidx] != nil + deleted := v[lidx] + resource.MarkStale(deleted) + wasDeleted := deleted != nil v[lidx] = nil isEmpty := true for _, vv := range v { @@ -681,10 +691,11 @@ func (s *contentNodeShifter) Delete(n contentNodeI, dimension doctree.Dimension) break } } - return wasDeleted, isEmpty + return deleted, wasDeleted, isEmpty case resourceSources: - resource.MarkStale(v[lidx]) - wasDeleted := v[lidx] != nil + deleted := v[lidx] + resource.MarkStale(deleted) + wasDeleted := deleted != nil v[lidx] = nil isEmpty := true for _, vv := range v { @@ -693,19 +704,19 @@ func (s *contentNodeShifter) Delete(n contentNodeI, dimension doctree.Dimension) break } } - return wasDeleted, isEmpty + return deleted, wasDeleted, isEmpty case *resourceSource: if lidx != v.LangIndex() { - return false, false + return nil, false, false } resource.MarkStale(v) - return true, true + return v, true, true case *pageState: if lidx != v.s.languagei { - return false, false + return nil, false, false } resource.MarkStale(v) - return true, true + return v, true, true default: panic(fmt.Sprintf("unknown type %T", n)) } @@ -778,7 +789,7 @@ func (s *contentNodeShifter) ForEeachInDimension(n contentNodeI, d int, f func(c } } -func (s *contentNodeShifter) InsertInto(old, new contentNodeI, dimension doctree.Dimension) contentNodeI { +func (s *contentNodeShifter) InsertInto(old, new contentNodeI, dimension doctree.Dimension) (contentNodeI, contentNodeI, bool) { langi := dimension[doctree.DimensionLanguage.Index()] switch vv := old.(type) { case *pageState: @@ -787,37 +798,39 @@ func (s *contentNodeShifter) InsertInto(old, new contentNodeI, dimension doctree panic(fmt.Sprintf("unknown type %T", new)) } if vv.s.languagei == newp.s.languagei && newp.s.languagei == langi { - return new + return new, vv, true } is := make(contentNodeIs, s.numLanguages) is[vv.s.languagei] = old is[langi] = new - return is + return is, old, false case contentNodeIs: + oldv := vv[langi] vv[langi] = new - return vv + return vv, oldv, oldv != nil case resourceSources: + oldv := vv[langi] vv[langi] = new.(*resourceSource) - return vv + return vv, oldv, oldv != nil case *resourceSource: newp, ok := new.(*resourceSource) if !ok { panic(fmt.Sprintf("unknown type %T", new)) } if vv.LangIndex() == newp.LangIndex() && newp.LangIndex() == langi { - return new + return new, vv, true } rs := make(resourceSources, s.numLanguages) rs[vv.LangIndex()] = vv rs[langi] = newp - return rs + return rs, vv, false default: panic(fmt.Sprintf("unknown type %T", old)) } } -func (s *contentNodeShifter) Insert(old, new contentNodeI) contentNodeI { +func (s *contentNodeShifter) Insert(old, new contentNodeI) (contentNodeI, contentNodeI, bool) { switch vv := old.(type) { case *pageState: newp, ok := new.(*pageState) @@ -828,12 +841,12 @@ func (s *contentNodeShifter) Insert(old, new contentNodeI) contentNodeI { if newp != old { resource.MarkStale(old) } - return new + return new, vv, true } is := make(contentNodeIs, s.numLanguages) is[newp.s.languagei] = new is[vv.s.languagei] = old - return is + return is, old, false case contentNodeIs: newp, ok := new.(*pageState) if !ok { @@ -844,7 +857,7 @@ func (s *contentNodeShifter) Insert(old, new contentNodeI) contentNodeI { resource.MarkStale(oldp) } vv[newp.s.languagei] = new - return vv + return vv, oldp, oldp != nil case *resourceSource: newp, ok := new.(*resourceSource) if !ok { @@ -854,12 +867,12 @@ func (s *contentNodeShifter) Insert(old, new contentNodeI) contentNodeI { if vv != newp { resource.MarkStale(vv) } - return new + return new, vv, true } rs := make(resourceSources, s.numLanguages) rs[newp.LangIndex()] = newp rs[vv.LangIndex()] = vv - return rs + return rs, vv, false case resourceSources: newp, ok := new.(*resourceSource) if !ok { @@ -870,7 +883,7 @@ func (s *contentNodeShifter) Insert(old, new contentNodeI) contentNodeI { resource.MarkStale(oldp) } vv[newp.LangIndex()] = newp - return vv + return vv, oldp, oldp != nil default: panic(fmt.Sprintf("unknown type %T", old)) } @@ -890,6 +903,8 @@ func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) * cacheContentPlain: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentPlainPlainWords]](mcache, fmt.Sprintf("/cont/pla/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), contentTableOfContents: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentTableOfContents]](mcache, fmt.Sprintf("/cont/toc/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), + contentDataFileSeenItems: maps.NewCache[string, map[uint64]bool](), + cfg: contentMapConfig{ lang: s.Lang(), taxonomyConfig: taxonomiesConfig.Values(), @@ -960,8 +975,6 @@ type contentTreeReverseIndexMap struct { type sitePagesAssembler struct { *Site - watching bool - incomingChanges *whatChanged assembleChanges *whatChanged ctx context.Context } @@ -1080,6 +1093,7 @@ func (h *HugoSites) resolveAndClearStateForIdentities( // 1. Handle the cache busters first, as those may produce identities for the page reset step. // 2. Then reset the page outputs, which may mark some resources as stale. // 3. Then GC the cache. + // TOOD1 if cachebuster != nil { if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) { ll := l.WithField("substep", "gc dynacache cachebuster") @@ -1125,6 +1139,33 @@ func (h *HugoSites) resolveAndClearStateForIdentities( } changes = changes[:n] + if h.pageTrees.treePagesFromTemplateAdapters.LenRaw() > 0 { + if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) { + ll := l.WithField("substep", "resolve content adapter change set").WithField("changes", len(changes)) + checkedCount := 0 + matchCount := 0 + depsFinder := identity.NewFinder(identity.FinderConfig{}) + + h.pageTrees.treePagesFromTemplateAdapters.WalkPrefixRaw(doctree.LockTypeRead, "", + func(s string, n *pagesfromdata.PagesFromTemplate) (bool, error) { + for _, id := range changes { + checkedCount++ + if r := depsFinder.Contains(id, n.DependencyManager, 2); r > identity.FinderNotFound { + n.MarkStale() + matchCount++ + break + } + } + return false, nil + }) + + ll = ll.WithField("checked", checkedCount).WithField("matches", matchCount) + return ll, nil + }); err != nil { + return err + } + } + if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) { // changesLeft: The IDs that the pages is dependent on. // changesRight: The IDs that the pages depend on. @@ -1269,6 +1310,7 @@ func (sa *sitePagesAssembler) applyAggregates() error { rw := pw.Extend() rw.Tree = sa.pageMap.treeResources sa.lastmod = time.Time{} + rebuild := sa.s.h.isRebuild() pw.Handle = func(keyPage string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { pageBundle := n.(*pageState) @@ -1289,7 +1331,7 @@ func (sa *sitePagesAssembler) applyAggregates() error { // Home page gets it's cascade from the site config. cascade = sa.conf.Cascade.Config - if pageBundle.m.pageConfig.Cascade == nil { + if pageBundle.m.pageConfig.CascadeCompiled == nil { // Pass the site cascade downwards. pw.WalkContext.Data().Insert(keyPage, cascade) } @@ -1300,18 +1342,20 @@ func (sa *sitePagesAssembler) applyAggregates() error { } } - if (pageBundle.IsHome() || pageBundle.IsSection()) && pageBundle.m.setMetaPostCount > 0 { - oldDates := pageBundle.m.pageConfig.Dates + if rebuild { + if (pageBundle.IsHome() || pageBundle.IsSection()) && pageBundle.m.setMetaPostCount > 0 { + oldDates := pageBundle.m.pageConfig.Dates - // We need to wait until after the walk to determine if any of the dates have changed. - pw.WalkContext.AddPostHook( - func() error { - if oldDates != pageBundle.m.pageConfig.Dates { - sa.assembleChanges.Add(pageBundle) - } - return nil - }, - ) + // We need to wait until after the walk to determine if any of the dates have changed. + pw.WalkContext.AddPostHook( + func() error { + if oldDates != pageBundle.m.pageConfig.Dates { + sa.assembleChanges.Add(pageBundle) + } + return nil + }, + ) + } } // Combine the cascade map with front matter. @@ -1321,15 +1365,15 @@ func (sa *sitePagesAssembler) applyAggregates() error { // We receive cascade values from above. If this leads to a change compared // to the previous value, we need to mark the page and its dependencies as changed. - if pageBundle.m.setMetaPostCascadeChanged { + if rebuild && pageBundle.m.setMetaPostCascadeChanged { sa.assembleChanges.Add(pageBundle) } const eventName = "dates" if n.isContentNodeBranch() { - if pageBundle.m.pageConfig.Cascade != nil { + if pageBundle.m.pageConfig.CascadeCompiled != nil { // Pass it down. - pw.WalkContext.Data().Insert(keyPage, pageBundle.m.pageConfig.Cascade) + pw.WalkContext.Data().Insert(keyPage, pageBundle.m.pageConfig.CascadeCompiled) } wasZeroDates := pageBundle.m.pageConfig.Dates.IsAllDatesZero() @@ -1430,9 +1474,9 @@ func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error { p := n.(*pageState) if p.Kind() != kinds.KindTerm { // The other kinds were handled in applyAggregates. - if p.m.pageConfig.Cascade != nil { + if p.m.pageConfig.CascadeCompiled != nil { // Pass it down. - pw.WalkContext.Data().Insert(s, p.m.pageConfig.Cascade) + pw.WalkContext.Data().Insert(s, p.m.pageConfig.CascadeCompiled) } } @@ -1553,7 +1597,7 @@ func (sa *sitePagesAssembler) assembleTermsAndTranslations() error { singular: viewName.singular, s: sa.Site, pathInfo: pi, - pageMetaParams: pageMetaParams{ + pageMetaParams: &pageMetaParams{ pageConfig: &pagemeta.PageConfig{ Kind: kinds.KindTerm, }, @@ -1615,13 +1659,13 @@ func (sa *sitePagesAssembler) assembleResources() error { targetPaths := ps.targetPaths() baseTarget := targetPaths.SubResourceBaseTarget duplicateResourceFiles := true - if ps.m.pageConfig.IsGoldmark { + if ps.m.pageConfig.ContentMediaType.IsMarkdown() { duplicateResourceFiles = ps.s.ContentSpec.Converters.GetMarkupConfig().Goldmark.DuplicateResourceFiles } duplicateResourceFiles = duplicateResourceFiles || ps.s.Conf.IsMultihost() - sa.pageMap.forEachResourceInPage( + err := sa.pageMap.forEachResourceInPage( ps, lockType, !duplicateResourceFiles, func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { @@ -1647,6 +1691,23 @@ func (sa *sitePagesAssembler) assembleResources() error { } + if rs.rc != nil && rs.rc.Content.IsResourceValue() { + if rs.rc.Name == "" { + rs.rc.Name = relPathOriginal + } + r, err := ps.m.s.ResourceSpec.NewResourceWrapperFromResourceConfig(rs.rc) + if err != nil { + return false, err + } + rs.r = r + return false, nil + } + + var mt media.Type + if rs.rc != nil { + mt = rs.rc.ContentMediaType + } + rd := resources.ResourceSourceDescriptor{ OpenReadSeekCloser: rs.opener, Path: rs.path, @@ -1657,8 +1718,23 @@ func (sa *sitePagesAssembler) assembleResources() error { BasePathTargetPath: baseTarget, NameNormalized: relPath, NameOriginal: relPathOriginal, + MediaType: mt, LazyPublish: !ps.m.pageConfig.Build.PublishResources, } + + if rs.rc != nil { + rc := rs.rc + rd.OpenReadSeekCloser = rc.Content.ValueAsOpenReadSeekCloser() + if rc.Name != "" { + rd.NameNormalized = rc.Name + rd.NameOriginal = rc.Name + } + if rc.Title != "" { + rd.Title = rc.Title + } + rd.Params = rc.Params + } + r, err := ps.m.s.ResourceSpec.NewResource(rd) if err != nil { return false, err @@ -1668,7 +1744,7 @@ func (sa *sitePagesAssembler) assembleResources() error { }, ) - return false, nil + return false, err }, } @@ -1775,7 +1851,7 @@ func (sa *sitePagesAssembler) addStandalonePages() error { m := &pageMeta{ s: s, pathInfo: s.Conf.PathParser().Parse(files.ComponentFolderContent, key+f.MediaType.FirstSuffix.FullSuffix), - pageMetaParams: pageMetaParams{ + pageMetaParams: &pageMetaParams{ pageConfig: &pagemeta.PageConfig{ Kind: kind, }, @@ -1893,7 +1969,7 @@ func (sa *sitePagesAssembler) addMissingRootSections() error { m := &pageMeta{ s: sa.Site, pathInfo: p, - pageMetaParams: pageMetaParams{ + pageMetaParams: &pageMetaParams{ pageConfig: &pagemeta.PageConfig{ Kind: kinds.KindHome, }, @@ -1903,7 +1979,7 @@ func (sa *sitePagesAssembler) addMissingRootSections() error { if err != nil { return err } - w.Tree.InsertWithLock(p.Base(), n) + w.Tree.InsertIntoValuesDimensionWithLock(p.Base(), n) sa.home = n } @@ -1926,7 +2002,7 @@ func (sa *sitePagesAssembler) addMissingTaxonomies() error { m := &pageMeta{ s: sa.Site, pathInfo: sa.Conf.PathParser().Parse(files.ComponentFolderContent, key+"/_index.md"), - pageMetaParams: pageMetaParams{ + pageMetaParams: &pageMetaParams{ pageConfig: &pagemeta.PageConfig{ Kind: kinds.KindTaxonomy, }, diff --git a/hugolib/doctree/nodeshiftree_test.go b/hugolib/doctree/nodeshiftree_test.go index 313be0bc4..ac89037ac 100644 --- a/hugolib/doctree/nodeshiftree_test.go +++ b/hugolib/doctree/nodeshiftree_test.go @@ -173,7 +173,7 @@ func TestTreeInsert(t *testing.T) { c.Assert(tree.Get("/notfound"), qt.IsNil) ab2 := &testValue{ID: "/a/b", Lang: 0} - v, ok := tree.InsertIntoValuesDimension("/a/b", ab2) + v, _, ok := tree.InsertIntoValuesDimension("/a/b", ab2) c.Assert(ok, qt.IsTrue) c.Assert(v, qt.DeepEquals, ab2) @@ -239,16 +239,16 @@ func (s *testShifter) ForEeachInDimension(n *testValue, d int, f func(n *testVal f(n) } -func (s *testShifter) Insert(old, new *testValue) *testValue { - return new +func (s *testShifter) Insert(old, new *testValue) (*testValue, *testValue, bool) { + return new, old, true } -func (s *testShifter) InsertInto(old, new *testValue, dimension doctree.Dimension) *testValue { - return new +func (s *testShifter) InsertInto(old, new *testValue, dimension doctree.Dimension) (*testValue, *testValue, bool) { + return new, old, true } -func (s *testShifter) Delete(n *testValue, dimension doctree.Dimension) (bool, bool) { - return true, true +func (s *testShifter) Delete(n *testValue, dimension doctree.Dimension) (*testValue, bool, bool) { + return nil, true, true } func (s *testShifter) Shift(n *testValue, dimension doctree.Dimension, exact bool) (*testValue, bool, doctree.DimensionFlag) { diff --git a/hugolib/doctree/nodeshifttree.go b/hugolib/doctree/nodeshifttree.go index 1c1175305..36382c2d7 100644 --- a/hugolib/doctree/nodeshifttree.go +++ b/hugolib/doctree/nodeshifttree.go @@ -38,16 +38,18 @@ type ( // Insert inserts new into the tree into the dimension it provides. // It may replace old. - // It returns a T (can be the same as old). - Insert(old, new T) T + // It returns the updated and existing T + // and a bool indicating if an existing record is updated. + Insert(old, new T) (T, T, bool) // Insert inserts new into the given dimension. // It may replace old. - // It returns a T (can be the same as old). - InsertInto(old, new T, dimension Dimension) T + // It returns the updated and existing T + // and a bool indicating if an existing record is updated. + InsertInto(old, new T, dimension Dimension) (T, T, bool) - // Delete deletes T from the given dimension and returns whether the dimension was deleted and if it's empty after the delete. - Delete(v T, dimension Dimension) (bool, bool) + // Delete deletes T from the given dimension and returns the deleted T and whether the dimension was deleted and if it's empty after the delete. + Delete(v T, dimension Dimension) (T, bool, bool) // Shift shifts T into the given dimension // and returns the shifted T and a bool indicating if the shift was successful and @@ -81,7 +83,11 @@ func New[T any](cfg Config[T]) *NodeShiftTree[T] { } } -func (r *NodeShiftTree[T]) Delete(key string) { +func (r *NodeShiftTree[T]) Delete(key string) (T, bool) { + return r.delete(key) +} + +func (r *NodeShiftTree[T]) DeleteRaw(key string) { r.delete(key) } @@ -103,23 +109,24 @@ func (r *NodeShiftTree[T]) DeletePrefix(prefix string) int { return false }) for _, key := range keys { - if ok := r.delete(key); ok { + if _, ok := r.delete(key); ok { count++ } } return count } -func (r *NodeShiftTree[T]) delete(key string) bool { +func (r *NodeShiftTree[T]) delete(key string) (T, bool) { var wasDeleted bool + var deleted T if v, ok := r.tree.Get(key); ok { var isEmpty bool - wasDeleted, isEmpty = r.shifter.Delete(v.(T), r.dims) + deleted, wasDeleted, isEmpty = r.shifter.Delete(v.(T), r.dims) if isEmpty { r.tree.Delete(key) } } - return wasDeleted + return deleted, wasDeleted } func (t *NodeShiftTree[T]) DeletePrefixAll(prefix string) int { @@ -141,22 +148,33 @@ func (t *NodeShiftTree[T]) Increment(d int) *NodeShiftTree[T] { return t.Shape(d, t.dims[d]+1) } -func (r *NodeShiftTree[T]) InsertIntoCurrentDimension(s string, v T) (T, bool) { +func (r *NodeShiftTree[T]) InsertIntoCurrentDimension(s string, v T) (T, T, bool) { s = mustValidateKey(cleanKey(s)) + var ( + updated bool + existing T + ) if vv, ok := r.tree.Get(s); ok { - v = r.shifter.InsertInto(vv.(T), v, r.dims) + v, existing, updated = r.shifter.InsertInto(vv.(T), v, r.dims) } r.tree.Insert(s, v) - return v, true + return v, existing, updated } -func (r *NodeShiftTree[T]) InsertIntoValuesDimension(s string, v T) (T, bool) { +// InsertIntoValuesDimension inserts v into the tree at the given key and the +// dimension defined by the value. +// It returns the updated and existing T and a bool indicating if an existing record is updated. +func (r *NodeShiftTree[T]) InsertIntoValuesDimension(s string, v T) (T, T, bool) { s = mustValidateKey(cleanKey(s)) + var ( + updated bool + existing T + ) if vv, ok := r.tree.Get(s); ok { - v = r.shifter.Insert(vv.(T), v) + v, existing, updated = r.shifter.Insert(vv.(T), v) } r.tree.Insert(s, v) - return v, true + return v, existing, updated } func (r *NodeShiftTree[T]) InsertRawWithLock(s string, v any) (any, bool) { @@ -165,7 +183,8 @@ func (r *NodeShiftTree[T]) InsertRawWithLock(s string, v any) (any, bool) { return r.tree.Insert(s, v) } -func (r *NodeShiftTree[T]) InsertWithLock(s string, v T) (T, bool) { +// It returns the updated and existing T and a bool indicating if an existing record is updated. +func (r *NodeShiftTree[T]) InsertIntoValuesDimensionWithLock(s string, v T) (T, T, bool) { r.mu.Lock() defer r.mu.Unlock() return r.InsertIntoValuesDimension(s, v) diff --git a/hugolib/doctree/simpletree.go b/hugolib/doctree/simpletree.go index 811a7ff80..2193c08f6 100644 --- a/hugolib/doctree/simpletree.go +++ b/hugolib/doctree/simpletree.go @@ -28,12 +28,12 @@ type Tree[T any] interface { } // NewSimpleTree creates a new SimpleTree. -func NewSimpleTree[T any]() *SimpleTree[T] { +func NewSimpleTree[T comparable]() *SimpleTree[T] { return &SimpleTree[T]{tree: radix.New()} } // SimpleTree is a thread safe radix tree that holds T. -type SimpleTree[T any] struct { +type SimpleTree[T comparable] struct { mu sync.RWMutex tree *radix.Tree zero T @@ -67,16 +67,23 @@ func (tree *SimpleTree[T]) Insert(s string, v T) T { return v } -func (tree *SimpleTree[T]) WalkPrefix(lockType LockType, s string, f func(s string, v T) (bool, error)) error { +func (tree *SimpleTree[T]) Lock(lockType LockType) func() { switch lockType { case LockTypeNone: + return func() {} case LockTypeRead: tree.mu.RLock() - defer tree.mu.RUnlock() + return tree.mu.RUnlock case LockTypeWrite: tree.mu.Lock() - defer tree.mu.Unlock() + return tree.mu.Unlock } + return func() {} +} + +func (tree *SimpleTree[T]) WalkPrefix(lockType LockType, s string, f func(s string, v T) (bool, error)) error { + commit := tree.Lock(lockType) + defer commit() var err error tree.tree.WalkPrefix(s, func(s string, v any) bool { var b bool diff --git a/hugolib/doctree/support.go b/hugolib/doctree/support.go index 8083df127..adcc3b06c 100644 --- a/hugolib/doctree/support.go +++ b/hugolib/doctree/support.go @@ -113,7 +113,7 @@ type LockType int // MutableTree is a tree that can be modified. type MutableTree interface { - Delete(key string) + DeleteRaw(key string) DeleteAll(key string) DeletePrefix(prefix string) int DeletePrefixAll(prefix string) int @@ -140,9 +140,9 @@ var _ MutableTree = MutableTrees(nil) type MutableTrees []MutableTree -func (t MutableTrees) Delete(key string) { +func (t MutableTrees) DeleteRaw(key string) { for _, tree := range t { - tree.Delete(key) + tree.DeleteRaw(key) } } diff --git a/hugolib/doctree/treeshifttree.go b/hugolib/doctree/treeshifttree.go index f8a6d360b..cd13b9f61 100644 --- a/hugolib/doctree/treeshifttree.go +++ b/hugolib/doctree/treeshifttree.go @@ -15,18 +15,21 @@ package doctree var _ Tree[string] = (*TreeShiftTree[string])(nil) -type TreeShiftTree[T any] struct { +type TreeShiftTree[T comparable] struct { // This tree is shiftable in one dimension. d int // The value of the current dimension. v int + // The zero value of T. + zero T + // Will be of length equal to the length of the dimension. trees []*SimpleTree[T] } -func NewTreeShiftTree[T any](d, length int) *TreeShiftTree[T] { +func NewTreeShiftTree[T comparable](d, length int) *TreeShiftTree[T] { if length <= 0 { panic("length must be > 0") } @@ -52,6 +55,17 @@ func (t *TreeShiftTree[T]) Get(s string) T { return t.trees[t.v].Get(s) } +func (t *TreeShiftTree[T]) DeleteAllFunc(s string, f func(s string, v T) bool) { + for _, tt := range t.trees { + if v := tt.Get(s); v != t.zero { + if f(s, v) { + // Delete. + tt.tree.Delete(s) + } + } + } +} + func (t *TreeShiftTree[T]) LongestPrefix(s string) (string, T) { return t.trees[t.v].LongestPrefix(s) } @@ -60,42 +74,41 @@ func (t *TreeShiftTree[T]) Insert(s string, v T) T { return t.trees[t.v].Insert(s, v) } +func (t *TreeShiftTree[T]) Lock(lockType LockType) func() { + return t.trees[t.v].Lock(lockType) +} + func (t *TreeShiftTree[T]) WalkPrefix(lockType LockType, s string, f func(s string, v T) (bool, error)) error { return t.trees[t.v].WalkPrefix(lockType, s, f) } -func (t *TreeShiftTree[T]) Delete(key string) { +func (t *TreeShiftTree[T]) WalkPrefixRaw(lockType LockType, s string, f func(s string, v T) (bool, error)) error { for _, tt := range t.trees { - tt.tree.Delete(key) + if err := tt.WalkPrefix(lockType, s, f); err != nil { + return err + } } + return nil } -func (t *TreeShiftTree[T]) DeletePrefix(prefix string) int { +func (t *TreeShiftTree[T]) LenRaw() int { var count int for _, tt := range t.trees { - count += tt.tree.DeletePrefix(prefix) + count += tt.tree.Len() } return count } -func (t *TreeShiftTree[T]) Lock(writable bool) (commit func()) { - if writable { - for _, tt := range t.trees { - tt.mu.Lock() - } - return func() { - for _, tt := range t.trees { - tt.mu.Unlock() - } - } +func (t *TreeShiftTree[T]) Delete(key string) { + for _, tt := range t.trees { + tt.tree.Delete(key) } +} +func (t *TreeShiftTree[T]) DeletePrefix(prefix string) int { + var count int for _, tt := range t.trees { - tt.mu.RLock() - } - return func() { - for _, tt := range t.trees { - tt.mu.RUnlock() - } + count += tt.tree.DeletePrefix(prefix) } + return count } diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index cf939ba92..61a07812d 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -111,6 +111,24 @@ func (h *HugoSites) ShouldSkipFileChangeEvent(ev fsnotify.Event) bool { return h.skipRebuildForFilenames[ev.Name] } +func (h *HugoSites) isRebuild() bool { + return h.buildCounter.Load() > 0 +} + +func (h *HugoSites) resolveSite(lang string) *Site { + if lang == "" { + lang = h.Conf.DefaultContentLanguage() + } + + for _, s := range h.Sites { + if s.Lang() == lang { + return s + } + } + + return nil +} + // Only used in tests. type buildCounters struct { contentRenderCounter atomic.Uint64 @@ -479,6 +497,7 @@ func (h *HugoSites) loadData() error { hugofs.WalkwayConfig{ Fs: h.PathSpec.BaseFs.Data.Fs, IgnoreFile: h.SourceSpec.IgnoreFile, + PathParser: h.Conf.PathParser(), WalkFn: func(path string, fi hugofs.FileMetaInfo) error { if fi.IsDir() { return nil diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 6a9afee99..4bea93039 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -30,6 +30,8 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugofs/files" "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/hugolib/doctree" + "github.com/gohugoio/hugo/hugolib/pagesfromdata" "github.com/gohugoio/hugo/hugolib/segments" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/output" @@ -41,6 +43,7 @@ import ( "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/para" "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/common/rungroup" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/page/siteidentities" @@ -96,6 +99,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { close(to) }(errCollector, errs) + for _, s := range h.Sites { + s.state = siteStateInit + } + if h.Metrics != nil { h.Metrics.Reset() } @@ -109,7 +116,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { conf := &config if conf.whatChanged == nil { // Assume everything has changed - conf.whatChanged = &whatChanged{contentChanged: true} + conf.whatChanged = &whatChanged{needsPagesAssembly: true} } var prepareErr error @@ -153,6 +160,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { } } + for _, s := range h.Sites { + s.state = siteStateReady + } + if prepareErr == nil { if err := h.render(infol, conf); err != nil { h.SendError(fmt.Errorf("render: %w", err)) @@ -213,7 +224,7 @@ func (h *HugoSites) initRebuild(config *BuildCfg) error { }) for _, s := range h.Sites { - s.resetBuildState(config.whatChanged.contentChanged) + s.resetBuildState(config.whatChanged.needsPagesAssembly) } h.reset(config) @@ -232,7 +243,7 @@ func (h *HugoSites) process(ctx context.Context, l logg.LevelLogger, config *Bui // This is a rebuild return h.processPartial(ctx, l, config, init, events) } - return h.processFull(ctx, l, *config) + return h.processFull(ctx, l, config) } // assemble creates missing sections, applies aggregate values (e.g. dates, cascading params), @@ -241,22 +252,24 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil l = l.WithField("step", "assemble") defer loggers.TimeTrackf(l, time.Now(), nil, "") - if !bcfg.whatChanged.contentChanged { + if !bcfg.whatChanged.needsPagesAssembly { + changes := bcfg.whatChanged.Drain() + if len(changes) > 0 { + if err := h.resolveAndClearStateForIdentities(ctx, l, nil, changes); err != nil { + return err + } + } return nil } h.translationKeyPages.Reset() assemblers := make([]*sitePagesAssembler, len(h.Sites)) // Changes detected during assembly (e.g. aggregate date changes) - assembleChanges := &whatChanged{ - identitySet: make(map[identity.Identity]bool), - } + for i, s := range h.Sites { assemblers[i] = &sitePagesAssembler{ Site: s, - watching: s.watching(), - incomingChanges: bcfg.whatChanged, - assembleChanges: assembleChanges, + assembleChanges: bcfg.whatChanged, ctx: ctx, } } @@ -272,7 +285,7 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil return err } - changes := assembleChanges.Changes() + changes := bcfg.whatChanged.Drain() // Changes from the assemble step (e.g. lastMod, cascade) needs a re-calculation // of what needs to be re-built. @@ -619,10 +632,10 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf logger := h.Log var ( - tmplAdded bool - tmplChanged bool - i18nChanged bool - contentChanged bool + tmplAdded bool + tmplChanged bool + i18nChanged bool + needsPagesAssemble bool ) changedPaths := struct { @@ -696,11 +709,33 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf switch pathInfo.Component() { case files.ComponentFolderContent: logger.Println("Source changed", pathInfo.Path()) - if ids := h.pageTrees.collectAndMarkStaleIdentities(pathInfo); len(ids) > 0 { - changes = append(changes, ids...) + isContentDataFile := pathInfo.IsContentData() + if !isContentDataFile { + if ids := h.pageTrees.collectAndMarkStaleIdentities(pathInfo); len(ids) > 0 { + changes = append(changes, ids...) + } + } else { + h.pageTrees.treePagesFromTemplateAdapters.DeleteAllFunc(pathInfo.Base(), + func(s string, n *pagesfromdata.PagesFromTemplate) bool { + changes = append(changes, n.DependencyManager) + + // Try to open the file to see if has been deleted. + f, err := n.GoTmplFi.Meta().Open() + if err == nil { + f.Close() + } + if err != nil { + // Remove all pages and resources below. + prefix := pathInfo.Base() + "/" + h.pageTrees.treePages.DeletePrefixAll(prefix) + h.pageTrees.resourceTrees.DeletePrefixAll(prefix) + changes = append(changes, identity.NewGlobIdentity(prefix+"*")) + } + return err != nil + }) } - contentChanged = true + needsPagesAssemble = true if config.RecentlyVisited != nil { // Fast render mode. Adding them to the visited queue @@ -714,7 +749,7 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf h.pageTrees.treeTaxonomyEntries.DeletePrefix("") - if delete { + if delete && !isContentDataFile { _, ok := h.pageTrees.treePages.LongestPrefixAll(pathInfo.Base()) if ok { h.pageTrees.treePages.DeleteAll(pathInfo.Base()) @@ -853,8 +888,8 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf resourceFiles := h.fileEventsContentPaths(addedOrChangedContent) changed := &whatChanged{ - contentChanged: contentChanged, - identitySet: make(identity.Identities), + needsPagesAssembly: needsPagesAssemble, + identitySet: make(identity.Identities), } changed.Add(changes...) @@ -876,10 +911,7 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf } } - // Removes duplicates. - changes = changed.identitySet.AsSlice() - - if err := h.resolveAndClearStateForIdentities(ctx, l, cacheBusterOr, changes); err != nil { + if err := h.resolveAndClearStateForIdentities(ctx, l, cacheBusterOr, changed.Drain()); err != nil { return err } @@ -907,7 +939,13 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf } if resourceFiles != nil { - if err := h.processFiles(ctx, l, *config, resourceFiles...); err != nil { + if err := h.processFiles(ctx, l, config, resourceFiles...); err != nil { + return err + } + } + + if h.isRebuild() { + if err := h.processContentAdaptersOnRebuild(ctx, config); err != nil { return err } } @@ -926,7 +964,7 @@ func (h *HugoSites) LogServerAddresses() { } } -func (h *HugoSites) processFull(ctx context.Context, l logg.LevelLogger, config BuildCfg) (err error) { +func (h *HugoSites) processFull(ctx context.Context, l logg.LevelLogger, config *BuildCfg) (err error) { if err = h.processFiles(ctx, l, config); err != nil { err = fmt.Errorf("readAndProcessContent: %w", err) return @@ -934,7 +972,49 @@ func (h *HugoSites) processFull(ctx context.Context, l logg.LevelLogger, config return err } -func (s *HugoSites) processFiles(ctx context.Context, l logg.LevelLogger, buildConfig BuildCfg, filenames ...pathChange) error { +func (s *Site) handleContentAdapterChanges(bi pagesfromdata.BuildInfo, buildConfig *BuildCfg) { + if !s.h.isRebuild() { + return + } + + if len(bi.ChangedIdentities) > 0 { + buildConfig.whatChanged.Add(bi.ChangedIdentities...) + buildConfig.whatChanged.needsPagesAssembly = true + } + + for _, p := range bi.DeletedPaths { + pp := path.Join(bi.Path.Base(), p) + if v, ok := s.pageMap.treePages.Delete(pp); ok { + buildConfig.whatChanged.Add(v.GetIdentity()) + } + } +} + +func (h *HugoSites) processContentAdaptersOnRebuild(ctx context.Context, buildConfig *BuildCfg) error { + g := rungroup.Run[*pagesfromdata.PagesFromTemplate](ctx, rungroup.Config[*pagesfromdata.PagesFromTemplate]{ + NumWorkers: h.numWorkers, + Handle: func(ctx context.Context, p *pagesfromdata.PagesFromTemplate) error { + bi, err := p.Execute(ctx) + if err != nil { + return err + } + s := p.Site.(*Site) + s.handleContentAdapterChanges(bi, buildConfig) + return nil + }, + }) + + h.pageTrees.treePagesFromTemplateAdapters.WalkPrefixRaw(doctree.LockTypeRead, "", func(key string, p *pagesfromdata.PagesFromTemplate) (bool, error) { + if p.StaleVersion() > 0 { + g.Enqueue(p) + } + return false, nil + }) + + return g.Wait() +} + +func (s *HugoSites) processFiles(ctx context.Context, l logg.LevelLogger, buildConfig *BuildCfg, filenames ...pathChange) error { if s.Deps == nil { panic("nil deps on site") } @@ -944,7 +1024,7 @@ func (s *HugoSites) processFiles(ctx context.Context, l logg.LevelLogger, buildC // For inserts, we can pick an arbitrary pageMap. pageMap := s.Sites[0].pageMap - c := newPagesCollector(ctx, s.h, sourceSpec, s.Log, l, pageMap, filenames) + c := newPagesCollector(ctx, s.h, sourceSpec, s.Log, l, pageMap, buildConfig, filenames) if err := c.Collect(); err != nil { return err diff --git a/hugolib/hugo_smoke_test.go b/hugolib/hugo_smoke_test.go index 42b0ab488..a4a5b7cbd 100644 --- a/hugolib/hugo_smoke_test.go +++ b/hugolib/hugo_smoke_test.go @@ -41,7 +41,7 @@ Home: {{ .Title }} IntegrationTestConfig{ T: t, TxtarString: files, - LogLevel: logg.LevelTrace, + // LogLevel: logg.LevelTrace, }, ).Build() diff --git a/hugolib/page.go b/hugolib/page.go index 028d3fa95..20751c57c 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -510,9 +510,15 @@ func (p *pageState) renderResources() error { continue } + if _, isWrapper := r.(resource.ResourceWrapper); isWrapper { + // Skip resources that are wrapped. + // These gets published on its own. + continue + } + src, ok := r.(resource.Source) if !ok { - initErr = fmt.Errorf("resource %T does not support resource.Source", src) + initErr = fmt.Errorf("resource %T does not support resource.Source", r) return } @@ -581,7 +587,11 @@ func (p *pageState) getPageInfoForError() string { func (p *pageState) getContentConverter() converter.Converter { var err error p.contentConverterInit.Do(func() { - markup := p.m.pageConfig.Markup + if p.m.pageConfig.ContentMediaType.IsZero() { + panic("ContentMediaType not set") + } + markup := p.m.pageConfig.ContentMediaType.SubType + if markup == "html" { // Only used for shortcode inner content. markup = "markdown" diff --git a/hugolib/page__content.go b/hugolib/page__content.go index 99ed824cd..1ef31f0f9 100644 --- a/hugolib/page__content.go +++ b/hugolib/page__content.go @@ -20,6 +20,7 @@ import ( "fmt" "html/template" "io" + "path/filepath" "strconv" "strings" "unicode/utf8" @@ -54,21 +55,31 @@ type pageContentReplacement struct { source pageparser.Item } -func (m *pageMeta) parseFrontMatter(h *HugoSites, pid uint64, sourceKey string) (*contentParseInfo, error) { - var openSource hugio.OpenReadSeekCloser - if m.f != nil { - meta := m.f.FileInfo().Meta() - openSource = func() (hugio.ReadSeekCloser, error) { - r, err := meta.Open() - if err != nil { - return nil, fmt.Errorf("failed to open file %q: %w", meta.Filename, err) +func (m *pageMeta) parseFrontMatter(h *HugoSites, pid uint64) (*contentParseInfo, error) { + var ( + sourceKey string + openSource hugio.OpenReadSeekCloser + hasContent = m.pageConfig.IsFromContentAdapter + ) + + if m.f != nil && !hasContent { + sourceKey = filepath.ToSlash(m.f.Filename()) + if !hasContent { + meta := m.f.FileInfo().Meta() + openSource = func() (hugio.ReadSeekCloser, error) { + r, err := meta.Open() + if err != nil { + return nil, fmt.Errorf("failed to open file %q: %w", meta.Filename, err) + } + return r, nil } - return r, nil } + } else if hasContent { + openSource = m.pageConfig.Content.ValueAsOpenReadSeekCloser() } if sourceKey == "" { - sourceKey = strconv.Itoa(int(pid)) + sourceKey = strconv.FormatUint(pid, 10) } pi := &contentParseInfo{ @@ -93,6 +104,11 @@ func (m *pageMeta) parseFrontMatter(h *HugoSites, pid uint64, sourceKey string) pi.itemsStep1 = items + if hasContent { + // No front matter. + return pi, nil + } + if err := pi.mapFrontMatter(source); err != nil { return nil, err } @@ -567,15 +583,14 @@ func (c *cachedContent) contentRendered(ctx context.Context, cp *pageContentOutp var result contentSummary // hasVariants bool if c.pi.hasSummaryDivider { - isHTML := cp.po.p.m.pageConfig.Markup == "html" - if isHTML { + if cp.po.p.m.pageConfig.ContentMediaType.IsHTML() { // Use the summary sections as provided by the user. i := bytes.Index(b, internalSummaryDividerPre) result.summary = helpers.BytesToHTML(b[:i]) b = b[i+len(internalSummaryDividerPre):] } else { - summary, content, err := splitUserDefinedSummaryAndContent(cp.po.p.m.pageConfig.Markup, b) + summary, content, err := splitUserDefinedSummaryAndContent(cp.po.p.m.pageConfig.Content.Markup, b) if err != nil { cp.po.p.s.Log.Errorf("Failed to set user defined summary for page %q: %s", cp.po.p.pathOrTitle(), err) } else { @@ -665,7 +680,7 @@ func (c *cachedContent) contentToC(ctx context.Context, cp *pageContentOutput) ( p.pageOutputTemplateVariationsState.Add(1) } - isHTML := cp.po.p.m.pageConfig.Markup == "html" + isHTML := cp.po.p.m.pageConfig.ContentMediaType.IsHTML() if !isHTML { createAndSetToC := func(tocProvider converter.TableOfContentsProvider) { @@ -788,7 +803,7 @@ func (c *cachedContent) contentPlain(ctx context.Context, cp *pageContentOutput) if err != nil { return nil, err } - html := cp.po.p.s.ContentSpec.TrimShortHTML(b.Bytes(), cp.po.p.m.pageConfig.Markup) + html := cp.po.p.s.ContentSpec.TrimShortHTML(b.Bytes(), cp.po.p.m.pageConfig.Content.Markup) result.summary = helpers.BytesToHTML(html) } else { var summary string diff --git a/hugolib/page__meta.go b/hugolib/page__meta.go index a88fe528d..fbc1a8aa1 100644 --- a/hugolib/page__meta.go +++ b/hugolib/page__meta.go @@ -54,7 +54,7 @@ type pageMeta struct { singular string // Set for kind == KindTerm and kind == KindTaxonomy. resource.Staler - pageMetaParams + *pageMetaParams pageMetaFrontMatter // Set for standalone pages, e.g. robotsTXT. @@ -100,7 +100,7 @@ type pageMetaFrontMatter struct { func (m *pageMetaParams) init(preserveOringal bool) { if preserveOringal { m.paramsOriginal = xmaps.Clone[maps.Params](m.pageConfig.Params) - m.cascadeOriginal = xmaps.Clone[map[page.PageMatcher]maps.Params](m.pageConfig.Cascade) + m.cascadeOriginal = xmaps.Clone[map[page.PageMatcher]maps.Params](m.pageConfig.CascadeCompiled) } } @@ -137,19 +137,19 @@ func (p *pageMeta) BundleType() string { } func (p *pageMeta) Date() time.Time { - return p.pageConfig.Date + return p.pageConfig.Dates.Date } func (p *pageMeta) PublishDate() time.Time { - return p.pageConfig.PublishDate + return p.pageConfig.Dates.PublishDate } func (p *pageMeta) Lastmod() time.Time { - return p.pageConfig.Lastmod + return p.pageConfig.Dates.Lastmod } func (p *pageMeta) ExpiryDate() time.Time { - return p.pageConfig.ExpiryDate + return p.pageConfig.Dates.ExpiryDate } func (p *pageMeta) Description() string { @@ -280,9 +280,6 @@ func (p *pageMeta) setMetaPre(pi *contentParseInfo, logger loggers.Logger, conf if frontmatter != nil { pcfg := p.pageConfig - if pcfg == nil { - panic("pageConfig not set") - } // Needed for case insensitive fetching of params values maps.PrepareParams(frontmatter) pcfg.Params = frontmatter @@ -293,7 +290,7 @@ func (p *pageMeta) setMetaPre(pi *contentParseInfo, logger loggers.Logger, conf if err != nil { return err } - pcfg.Cascade = cascade + pcfg.CascadeCompiled = cascade } // Look for path, lang and kind, all of which values we need early on. @@ -331,18 +328,18 @@ func (ps *pageState) setMetaPost(cascade map[page.PageMatcher]maps.Params) error ps.m.setMetaPostCount++ var cascadeHashPre uint64 if ps.m.setMetaPostCount > 1 { - cascadeHashPre = identity.HashUint64(ps.m.pageConfig.Cascade) - ps.m.pageConfig.Cascade = xmaps.Clone[map[page.PageMatcher]maps.Params](ps.m.cascadeOriginal) + cascadeHashPre = identity.HashUint64(ps.m.pageConfig.CascadeCompiled) + ps.m.pageConfig.CascadeCompiled = xmaps.Clone[map[page.PageMatcher]maps.Params](ps.m.cascadeOriginal) } // Apply cascades first so they can be overridden later. if cascade != nil { - if ps.m.pageConfig.Cascade != nil { + if ps.m.pageConfig.CascadeCompiled != nil { for k, v := range cascade { - vv, found := ps.m.pageConfig.Cascade[k] + vv, found := ps.m.pageConfig.CascadeCompiled[k] if !found { - ps.m.pageConfig.Cascade[k] = v + ps.m.pageConfig.CascadeCompiled[k] = v } else { // Merge for ck, cv := range v { @@ -352,18 +349,18 @@ func (ps *pageState) setMetaPost(cascade map[page.PageMatcher]maps.Params) error } } } - cascade = ps.m.pageConfig.Cascade + cascade = ps.m.pageConfig.CascadeCompiled } else { - ps.m.pageConfig.Cascade = cascade + ps.m.pageConfig.CascadeCompiled = cascade } } if cascade == nil { - cascade = ps.m.pageConfig.Cascade + cascade = ps.m.pageConfig.CascadeCompiled } if ps.m.setMetaPostCount > 1 { - ps.m.setMetaPostCascadeChanged = cascadeHashPre != identity.HashUint64(ps.m.pageConfig.Cascade) + ps.m.setMetaPostCascadeChanged = cascadeHashPre != identity.HashUint64(ps.m.pageConfig.CascadeCompiled) if !ps.m.setMetaPostCascadeChanged { // No changes, restore any value that may be changed by aggregation. @@ -404,11 +401,17 @@ func (p *pageState) setMetaPostParams() error { pm := p.m var mtime time.Time var contentBaseName string + var ext string + var isContentAdapter bool if p.File() != nil { + isContentAdapter = p.File().IsContentAdapter() contentBaseName = p.File().ContentBaseName() if p.File().FileInfo() != nil { mtime = p.File().FileInfo().ModTime() } + if !isContentAdapter { + ext = p.File().Ext() + } } var gitAuthorDate time.Time @@ -432,6 +435,11 @@ func (p *pageState) setMetaPostParams() error { p.s.Log.Errorf("Failed to handle dates for page %q: %s", p.pathOrTitle(), err) } + if isContentAdapter { + // Done. + return nil + } + var buildConfig any var isNewBuildKeyword bool if v, ok := pm.pageConfig.Params["_build"]; ok { @@ -460,8 +468,10 @@ params: var sitemapSet bool pcfg := pm.pageConfig - params := pcfg.Params + if params == nil { + panic("params not set for " + p.Title()) + } var draft, published, isCJKLanguage *bool var userParams map[string]any @@ -554,8 +564,8 @@ params: pcfg.Layout = cast.ToString(v) params[loki] = pcfg.Layout case "markup": - pcfg.Markup = cast.ToString(v) - params[loki] = pcfg.Markup + pcfg.Content.Markup = cast.ToString(v) + params[loki] = pcfg.Content.Markup case "weight": pcfg.Weight = cast.ToInt(v) params[loki] = pcfg.Weight @@ -605,7 +615,7 @@ params: } if handled { - pcfg.Resources = resources + pcfg.ResourcesMeta = resources break } fallthrough @@ -652,8 +662,6 @@ params: pcfg.Sitemap = p.s.conf.Sitemap } - pcfg.Markup = p.s.ContentSpec.ResolveMarkup(pcfg.Markup) - if draft != nil && published != nil { pcfg.Draft = *draft p.m.s.Log.Warnf("page %q has both draft and published settings in its frontmatter. Using draft.", p.File().Filename()) @@ -676,6 +684,14 @@ params: params["iscjklanguage"] = pcfg.IsCJKLanguage + if err := pcfg.Validate(false); err != nil { + return err + } + + if err := pcfg.Compile("", false, ext, p.s.Log, p.s.conf.MediaTypes.Config); err != nil { + return err + } + return nil } @@ -731,18 +747,16 @@ func (p *pageMeta) applyDefaultValues() error { (&p.pageConfig.Build).Disable() } - if p.pageConfig.Markup == "" { + if p.pageConfig.Content.Markup == "" { if p.File() != nil { // Fall back to file extension - p.pageConfig.Markup = p.s.ContentSpec.ResolveMarkup(p.File().Ext()) + p.pageConfig.Content.Markup = p.s.ContentSpec.ResolveMarkup(p.File().Ext()) } - if p.pageConfig.Markup == "" { - p.pageConfig.Markup = "markdown" + if p.pageConfig.Content.Markup == "" { + p.pageConfig.Content.Markup = "markdown" } } - p.pageConfig.IsGoldmark = p.s.ContentSpec.Converters.IsGoldmark(p.pageConfig.Markup) - if p.pageConfig.Title == "" && p.f == nil { switch p.Kind() { case kinds.KindHome: diff --git a/hugolib/page__new.go b/hugolib/page__new.go index ac3962883..04c68ba6a 100644 --- a/hugolib/page__new.go +++ b/hugolib/page__new.go @@ -15,7 +15,6 @@ package hugolib import ( "fmt" - "path/filepath" "sync" "sync/atomic" @@ -36,21 +35,17 @@ var pageIDCounter atomic.Uint64 func (h *HugoSites) newPage(m *pageMeta) (*pageState, *paths.Path, error) { m.Staler = &resources.AtomicStaler{} - if m.pageConfig == nil { - m.pageMetaParams = pageMetaParams{ - pageConfig: &pagemeta.PageConfig{ - Params: maps.Params{}, - }, + if m.pageMetaParams == nil { + m.pageMetaParams = &pageMetaParams{ + pageConfig: &pagemeta.PageConfig{}, } } - - var sourceKey string - if m.f != nil { - sourceKey = filepath.ToSlash(m.f.Filename()) + if m.pageConfig.Params == nil { + m.pageConfig.Params = maps.Params{} } pid := pageIDCounter.Add(1) - pi, err := m.parseFrontMatter(h, pid, sourceKey) + pi, err := m.parseFrontMatter(h, pid) if err != nil { return nil, nil, err } @@ -69,28 +64,40 @@ func (h *HugoSites) newPage(m *pageMeta) (*pageState, *paths.Path, error) { s := m.pageConfig.Path if !paths.HasExt(s) { var ( - isBranch bool - ext string = "md" + isBranch bool + isBranchSet bool + ext string = m.pageConfig.ContentMediaType.FirstSuffix.Suffix ) if pcfg.Kind != "" { isBranch = kinds.IsBranch(pcfg.Kind) - } else if m.pathInfo != nil { - isBranch = m.pathInfo.IsBranchBundle() - if m.pathInfo.Ext() != "" { - ext = m.pathInfo.Ext() - } - } else if m.f != nil { - pi := m.f.FileInfo().Meta().PathInfo - isBranch = pi.IsBranchBundle() - if pi.Ext() != "" { - ext = pi.Ext() + isBranchSet = true + } + + if !pcfg.IsFromContentAdapter { + if m.pathInfo != nil { + if !isBranchSet { + isBranch = m.pathInfo.IsBranchBundle() + } + if m.pathInfo.Ext() != "" { + ext = m.pathInfo.Ext() + } + } else if m.f != nil { + pi := m.f.FileInfo().Meta().PathInfo + if !isBranchSet { + isBranch = pi.IsBranchBundle() + } + if pi.Ext() != "" { + ext = pi.Ext() + } } } + if isBranch { s += "/_index." + ext } else { s += "/index." + ext } + } m.pathInfo = h.Conf.PathParser().Parse(files.ComponentFolderContent, s) } else if m.pathInfo == nil { @@ -112,23 +119,13 @@ func (h *HugoSites) newPage(m *pageMeta) (*pageState, *paths.Path, error) { } else if m.f != nil { meta := m.f.FileInfo().Meta() lang = meta.Lang - m.s = h.Sites[meta.LangIndex] } else { lang = m.pathInfo.Lang() } - if lang == "" { - lang = h.Conf.DefaultContentLanguage() - } - var found bool - for _, ss := range h.Sites { - if ss.Lang() == lang { - m.s = ss - found = true - break - } - } - if !found { + m.s = h.resolveSite(lang) + + if m.s == nil { return nil, fmt.Errorf("no site found for language %q", lang) } } diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go index 6b4b8f55e..5b039b91b 100644 --- a/hugolib/page__per_output.go +++ b/hugolib/page__per_output.go @@ -25,6 +25,8 @@ import ( "github.com/gohugoio/hugo/common/text" "github.com/gohugoio/hugo/common/types/hstring" "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/markup" + "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/parser/pageparser" "github.com/mitchellh/mapstructure" "github.com/spf13/cast" @@ -262,6 +264,9 @@ func (pco *pageContentOutput) RenderString(ctx context.Context, args ...any) (te if err := mapstructure.WeakDecode(m, &opts); err != nil { return "", fmt.Errorf("failed to decode options: %w", err) } + if opts.Markup != "" { + opts.Markup = markup.ResolveMarkup(opts.Markup) + } } contentToRenderv := args[sidx] @@ -283,7 +288,8 @@ func (pco *pageContentOutput) RenderString(ctx context.Context, args ...any) (te } conv := pco.po.p.getContentConverter() - if opts.Markup != "" && opts.Markup != pco.po.p.m.pageConfig.Markup { + + if opts.Markup != "" && opts.Markup != pco.po.p.m.pageConfig.ContentMediaType.SubType { var err error conv, err = pco.po.p.m.newContentConverter(pco.po.p, opts.Markup) if err != nil { @@ -376,7 +382,7 @@ func (pco *pageContentOutput) RenderString(ctx context.Context, args ...any) (te } if opts.Display == "inline" { - markup := pco.po.p.m.pageConfig.Markup + markup := pco.po.p.m.pageConfig.Content.Markup if opts.Markup != "" { markup = pco.po.p.s.ContentSpec.ResolveMarkup(opts.Markup) } @@ -657,7 +663,7 @@ func splitUserDefinedSummaryAndContent(markup string, c []byte) (summary []byte, startTag := "p" switch markup { - case "asciidocext": + case media.DefaultContentTypes.AsciiDoc.SubType: startTag = "div" } diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go index 1633feb3e..96c2c0f96 100644 --- a/hugolib/pages_capture.go +++ b/hugolib/pages_capture.go @@ -42,18 +42,20 @@ func newPagesCollector( logger loggers.Logger, infoLogger logg.LevelLogger, m *pageMap, + buildConfig *BuildCfg, ids []pathChange, ) *pagesCollector { return &pagesCollector{ - ctx: ctx, - h: h, - fs: sp.BaseFs.Content.Fs, - m: m, - sp: sp, - logger: logger, - infoLogger: infoLogger, - ids: ids, - seenDirs: make(map[string]bool), + ctx: ctx, + h: h, + fs: sp.BaseFs.Content.Fs, + m: m, + sp: sp, + logger: logger, + infoLogger: infoLogger, + buildConfig: buildConfig, + ids: ids, + seenDirs: make(map[string]bool), } } @@ -68,6 +70,8 @@ type pagesCollector struct { fs afero.Fs + buildConfig *BuildCfg + // List of paths that have changed. Used in partial builds. ids []pathChange seenDirs map[string]bool @@ -82,6 +86,8 @@ func (c *pagesCollector) Collect() (collectErr error) { var ( numWorkers = c.h.numWorkers numFilesProcessedTotal atomic.Uint64 + numPagesProcessedTotal atomic.Uint64 + numResourcesProcessed atomic.Uint64 numFilesProcessedLast uint64 fileBatchTimer = time.Now() fileBatchTimerMu sync.Mutex @@ -98,6 +104,8 @@ func (c *pagesCollector) Collect() (collectErr error) { logg.Fields{ logg.Field{Name: "files", Value: numFilesProcessedBatch}, logg.Field{Name: "files_total", Value: numFilesProcessedTotal.Load()}, + logg.Field{Name: "pages_total", Value: numPagesProcessedTotal.Load()}, + logg.Field{Name: "resources_total", Value: numResourcesProcessed.Load()}, }, "", ) @@ -113,10 +121,13 @@ func (c *pagesCollector) Collect() (collectErr error) { c.g = rungroup.Run[hugofs.FileMetaInfo](c.ctx, rungroup.Config[hugofs.FileMetaInfo]{ NumWorkers: numWorkers, Handle: func(ctx context.Context, fi hugofs.FileMetaInfo) error { - if err := c.m.AddFi(fi); err != nil { + numPages, numResources, err := c.m.AddFi(fi, c.buildConfig) + if err != nil { return hugofs.AddFileInfoToError(err, fi, c.fs) } numFilesProcessedTotal.Add(1) + numPagesProcessedTotal.Add(numPages) + numResourcesProcessed.Add(numResources) if numFilesProcessedTotal.Load()%1000 == 0 { logFilesProcessed(false) } @@ -243,6 +254,21 @@ func (c *pagesCollector) collectDirDir(path string, root hugofs.FileMetaInfo, in return nil, nil } + n := 0 + for _, fi := range readdir { + if fi.Meta().PathInfo.IsContentData() { + // _content.json + // These are not part of any bundle, so just add them directly and remove them from the readdir slice. + if err := c.g.Enqueue(fi); err != nil { + return nil, err + } + } else { + readdir[n] = fi + n++ + } + } + readdir = readdir[:n] + // Pick the first regular file. var first hugofs.FileMetaInfo for _, fi := range readdir { @@ -260,6 +286,7 @@ func (c *pagesCollector) collectDirDir(path string, root hugofs.FileMetaInfo, in // Any bundle file will always be first. firstPi := first.Meta().PathInfo + if firstPi == nil { panic(fmt.Sprintf("collectDirDir: no path info for %q", first.Meta().Filename)) } @@ -320,6 +347,7 @@ func (c *pagesCollector) collectDirDir(path string, root hugofs.FileMetaInfo, in Info: root, Fs: c.fs, IgnoreFile: c.h.SourceSpec.IgnoreFile, + PathParser: c.h.Conf.PathParser(), HookPre: preHook, HookPost: postHook, WalkFn: wfn, @@ -370,6 +398,7 @@ func (c *pagesCollector) handleBundleLeaf(dir, bundle hugofs.FileMetaInfo, inPat Info: dir, DirEntries: readdir, IgnoreFile: c.h.SourceSpec.IgnoreFile, + PathParser: c.h.Conf.PathParser(), WalkFn: walk, }) diff --git a/hugolib/pagesfromdata/pagesfromgotmpl.go b/hugolib/pagesfromdata/pagesfromgotmpl.go new file mode 100644 index 000000000..4107f9e33 --- /dev/null +++ b/hugolib/pagesfromdata/pagesfromgotmpl.go @@ -0,0 +1,331 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pagesfromdata + +import ( + "context" + "fmt" + "io" + "path/filepath" + + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/page/pagemeta" + "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/tpl" + "github.com/mitchellh/mapstructure" + "github.com/spf13/cast" +) + +type PagesFromDataTemplateContext interface { + // AddPage adds a new page to the site. + // The first return value will always be an empty string. + AddPage(any) (string, error) + + // AddResource adds a new resource to the site. + // The first return value will always be an empty string. + AddResource(any) (string, error) + + // The site to which the pages will be added. + Site() page.Site + + // The same template may be executed multiple times for multiple languages. + // The Store can be used to store state between these invocations. + Store() *maps.Scratch + + // By default, the template will be executed for the language + // defined by the _content.gotmpl file (e.g. its mount definition). + // This method can be used to activate the template for all languages. + // The return value will always be an empty string. + EnableAllLanguages() string +} + +var _ PagesFromDataTemplateContext = (*pagesFromDataTemplateContext)(nil) + +type pagesFromDataTemplateContext struct { + p *PagesFromTemplate +} + +func (p *pagesFromDataTemplateContext) toPathMap(v any) (string, map[string]any, error) { + m, err := maps.ToStringMapE(v) + if err != nil { + return "", nil, err + } + pathv, ok := m["path"] + if !ok { + return "", nil, fmt.Errorf("path not set") + } + path, err := cast.ToStringE(pathv) + if err != nil || path == "" { + return "", nil, fmt.Errorf("invalid path %q", path) + } + return path, m, nil +} + +func (p *pagesFromDataTemplateContext) AddPage(v any) (string, error) { + path, m, err := p.toPathMap(v) + if err != nil { + return "", err + } + + if !p.p.buildState.checkHasChangedAndSetSourceInfo(path, m) { + return "", nil + } + + pd := pagemeta.DefaultPageConfig + pd.IsFromContentAdapter = true + + if err := mapstructure.WeakDecode(m, &pd); err != nil { + return "", fmt.Errorf("failed to decode page map: %w", err) + } + + p.p.buildState.NumPagesAdded++ + + if err := pd.Validate(true); err != nil { + return "", err + } + + return "", p.p.HandlePage(p.p, &pd) +} + +func (p *pagesFromDataTemplateContext) AddResource(v any) (string, error) { + path, m, err := p.toPathMap(v) + if err != nil { + return "", err + } + + if !p.p.buildState.checkHasChangedAndSetSourceInfo(path, m) { + return "", nil + } + + var rd pagemeta.ResourceConfig + if err := mapstructure.WeakDecode(m, &rd); err != nil { + return "", err + } + + p.p.buildState.NumResourcesAdded++ + + if err := rd.Validate(); err != nil { + return "", err + } + + return "", p.p.HandleResource(p.p, &rd) +} + +func (p *pagesFromDataTemplateContext) Site() page.Site { + return p.p.Site +} + +func (p *pagesFromDataTemplateContext) Store() *maps.Scratch { + return p.p.store +} + +func (p *pagesFromDataTemplateContext) EnableAllLanguages() string { + p.p.buildState.EnableAllLanguages = true + return "" +} + +func NewPagesFromTemplate(opts PagesFromTemplateOptions) *PagesFromTemplate { + return &PagesFromTemplate{ + PagesFromTemplateOptions: opts, + PagesFromTemplateDeps: opts.DepsFromSite(opts.Site), + buildState: &BuildState{ + sourceInfosCurrent: maps.NewCache[string, *sourceInfo](), + }, + store: maps.NewScratch(), + } +} + +type PagesFromTemplateOptions struct { + Site page.Site + DepsFromSite func(page.Site) PagesFromTemplateDeps + + DependencyManager identity.Manager + + Watching bool + + HandlePage func(pt *PagesFromTemplate, p *pagemeta.PageConfig) error + HandleResource func(pt *PagesFromTemplate, p *pagemeta.ResourceConfig) error + + GoTmplFi hugofs.FileMetaInfo +} + +type PagesFromTemplateDeps struct { + TmplFinder tpl.TemplateParseFinder + TmplExec tpl.TemplateExecutor +} + +var _ resource.Staler = (*PagesFromTemplate)(nil) + +type PagesFromTemplate struct { + PagesFromTemplateOptions + PagesFromTemplateDeps + buildState *BuildState + store *maps.Scratch +} + +func (b *PagesFromTemplate) AddChange(id identity.Identity) { + b.buildState.ChangedIdentities = append(b.buildState.ChangedIdentities, id) +} + +func (b *PagesFromTemplate) MarkStale() { + b.buildState.StaleVersion++ +} + +func (b *PagesFromTemplate) StaleVersion() uint32 { + return b.buildState.StaleVersion +} + +type BuildInfo struct { + NumPagesAdded uint64 + NumResourcesAdded uint64 + EnableAllLanguages bool + ChangedIdentities []identity.Identity + DeletedPaths []string + Path *paths.Path +} + +type BuildState struct { + StaleVersion uint32 + + EnableAllLanguages bool + + // Paths deleted in the current build. + DeletedPaths []string + + // Changed identities in the current build. + ChangedIdentities []identity.Identity + + NumPagesAdded uint64 + NumResourcesAdded uint64 + + sourceInfosCurrent *maps.Cache[string, *sourceInfo] + sourceInfosPrevious *maps.Cache[string, *sourceInfo] +} + +func (b *BuildState) hash(v any) uint64 { + return identity.HashUint64(v) +} + +func (b *BuildState) checkHasChangedAndSetSourceInfo(changedPath string, v any) bool { + h := b.hash(v) + si, found := b.sourceInfosPrevious.Get(changedPath) + if found { + b.sourceInfosCurrent.Set(changedPath, si) + if si.hash == h { + return false + } + } else { + si = &sourceInfo{} + b.sourceInfosCurrent.Set(changedPath, si) + } + si.hash = h + return true +} + +func (b *BuildState) resolveDeletedPaths() { + if b.sourceInfosPrevious == nil { + b.DeletedPaths = nil + return + } + var paths []string + b.sourceInfosPrevious.ForEeach(func(k string, _ *sourceInfo) { + if _, found := b.sourceInfosCurrent.Get(k); !found { + paths = append(paths, k) + } + }) + + b.DeletedPaths = paths +} + +func (b *BuildState) PrepareNextBuild() { + b.sourceInfosPrevious = b.sourceInfosCurrent + b.sourceInfosCurrent = maps.NewCache[string, *sourceInfo]() + b.StaleVersion = 0 + b.DeletedPaths = nil + b.ChangedIdentities = nil + b.NumPagesAdded = 0 + b.NumResourcesAdded = 0 +} + +type sourceInfo struct { + hash uint64 +} + +func (p PagesFromTemplate) CloneForSite(s page.Site) *PagesFromTemplate { + // We deliberately make them share the same DepenencyManager and Store. + p.PagesFromTemplateOptions.Site = s + p.PagesFromTemplateDeps = p.PagesFromTemplateOptions.DepsFromSite(s) + p.buildState = &BuildState{ + sourceInfosCurrent: maps.NewCache[string, *sourceInfo](), + } + return &p +} + +func (p PagesFromTemplate) CloneForGoTmpl(fi hugofs.FileMetaInfo) *PagesFromTemplate { + p.PagesFromTemplateOptions.GoTmplFi = fi + return &p +} + +func (p *PagesFromTemplate) GetDependencyManagerForScope(scope int) identity.Manager { + return p.DependencyManager +} + +func (p *PagesFromTemplate) Execute(ctx context.Context) (BuildInfo, error) { + defer func() { + p.buildState.PrepareNextBuild() + }() + + f, err := p.GoTmplFi.Meta().Open() + if err != nil { + return BuildInfo{}, err + } + defer f.Close() + + tmpl, err := p.TmplFinder.Parse(filepath.ToSlash(p.GoTmplFi.Meta().Filename), helpers.ReaderToString(f)) + if err != nil { + return BuildInfo{}, err + } + + data := &pagesFromDataTemplateContext{ + p: p, + } + + ctx = tpl.Context.DependencyManagerScopedProvider.Set(ctx, p) + + if err := p.TmplExec.ExecuteWithContext(ctx, tmpl, io.Discard, data); err != nil { + return BuildInfo{}, err + } + + if p.Watching { + p.buildState.resolveDeletedPaths() + } + + bi := BuildInfo{ + NumPagesAdded: p.buildState.NumPagesAdded, + NumResourcesAdded: p.buildState.NumResourcesAdded, + EnableAllLanguages: p.buildState.EnableAllLanguages, + ChangedIdentities: p.buildState.ChangedIdentities, + DeletedPaths: p.buildState.DeletedPaths, + Path: p.GoTmplFi.Meta().PathInfo, + } + + return bi, nil +} + +////////////// diff --git a/hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go b/hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go new file mode 100644 index 000000000..60930321a --- /dev/null +++ b/hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go @@ -0,0 +1,479 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pagesfromdata_test + +import ( + "fmt" + "strings" + "testing" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/hugolib" + "github.com/gohugoio/hugo/markup/asciidocext" + "github.com/gohugoio/hugo/markup/pandoc" + "github.com/gohugoio/hugo/markup/rst" +) + +const filesPagesFromDataTempleBasic = ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "rss", "sitemap"] +baseURL = "https://example.com" +disableLiveReload = true +-- assets/a/pixel.png -- +iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg== +-- assets/mydata.yaml -- +p1: "p1" +draft: false +-- layouts/partials/get-value.html -- +{{ $val := "p1" }} +{{ return $val }} +-- layouts/_default/single.html -- +Single: {{ .Title }}|{{ .Content }}|Params: {{ .Params.param1 }}|Path: {{ .Path }}| +Dates: Date: {{ .Date.Format "2006-01-02" }}|Lastmod: {{ .Lastmod.Format "2006-01-02" }}|PublishDate: {{ .PublishDate.Format "2006-01-02" }}|ExpiryDate: {{ .ExpiryDate.Format "2006-01-02" }}| +Len Resources: {{ .Resources | len }} +Resources: {{ range .Resources }}RelPermalink: {{ .RelPermalink }}|Name: {{ .Name }}|Title: {{ .Title }}|Params: {{ .Params }}|{{ end }}$ +{{ with .Resources.Get "featured.png" }} +Featured Image: {{ .RelPermalink }}|{{ .Name }}| +{{ with .Resize "10x10" }} +Resized Featured Image: {{ .RelPermalink }}|{{ .Width }}| +{{ end}} +{{ end }} +-- layouts/_default/list.html -- +List: {{ .Title }}|{{ .Content }}| +RegularPagesRecursive: {{ range .RegularPagesRecursive }}{{ .Title }}:{{ .Path }}|{{ end }}$ +Sections: {{ range .Sections }}{{ .Title }}:{{ .Path }}|{{ end }}$ +-- content/docs/pfile.md -- +--- +title: "pfile" +date: 2023-03-01 +--- +Pfile Content +-- content/docs/_content.gotmpl -- +{{ $pixel := resources.Get "a/pixel.png" }} +{{ $dataResource := resources.Get "mydata.yaml" }} +{{ $data := $dataResource | transform.Unmarshal }} +{{ $pd := $data.p1 }} +{{ $pp := partial "get-value.html" }} +{{ $title := printf "%s:%s" $pd $pp }} +{{ $date := "2023-03-01" | time.AsTime }} +{{ $dates := dict "date" $date }} +{{ $contentMarkdown := dict "value" "**Hello World**" "mediaType" "text/markdown" }} +{{ $contentMarkdownDefault := dict "value" "**Hello World Default**" }} +{{ $contentHTML := dict "value" "<b>Hello World!</b> No **markdown** here." "mediaType" "text/html" }} +{{ $.AddPage (dict "kind" "page" "path" "P1" "title" $title "dates" $dates "content" $contentMarkdown "params" (dict "param1" "param1v" ) ) }} +{{ $.AddPage (dict "kind" "page" "path" "p2" "title" "p2title" "dates" $dates "content" $contentHTML ) }} +{{ $.AddPage (dict "kind" "page" "path" "p3" "title" "p3title" "dates" $dates "content" $contentMarkdownDefault "draft" false ) }} +{{ $.AddPage (dict "kind" "page" "path" "p4" "title" "p4title" "dates" $dates "content" $contentMarkdownDefault "draft" $data.draft ) }} + + +{{ $resourceContent := dict "value" $dataResource }} +{{ $.AddResource (dict "path" "p1/data1.yaml" "content" $resourceContent) }} +{{ $.AddResource (dict "path" "p1/mytext.txt" "content" (dict "value" "some text") "name" "textresource" "title" "My Text Resource" "params" (dict "param1" "param1v") )}} +{{ $.AddResource (dict "path" "p1/sub/mytex2.txt" "content" (dict "value" "some text") "title" "My Text Sub Resource" ) }} +{{ $.AddResource (dict "path" "P1/Sub/MyMixCaseText2.txt" "content" (dict "value" "some text") "title" "My Text Sub Mixed Case Path Resource" ) }} +{{ $.AddResource (dict "path" "p1/sub/data1.yaml" "content" $resourceContent "title" "Sub data") }} +{{ $resourceParams := dict "data2ParaM1" "data2Param1v" }} +{{ $.AddResource (dict "path" "p1/data2.yaml" "name" "data2.yaml" "title" "My data 2" "params" $resourceParams "content" $resourceContent) }} +{{ $.AddResource (dict "path" "p1/featuredimage.png" "name" "featured.png" "title" "My Featured Image" "params" $resourceParams "content" (dict "value" $pixel ))}} +` + +func TestPagesFromGoTmplMisc(t *testing.T) { + t.Parallel() + b := hugolib.Test(t, filesPagesFromDataTempleBasic) + b.AssertPublishDir(` +docs/p1/mytext.txt +docs/p1/sub/mytex2.tx +docs/p1/sub/mymixcasetext2.txt + `) + + // Page from markdown file. + b.AssertFileContent("public/docs/pfile/index.html", "Dates: Date: 2023-03-01|Lastmod: 2023-03-01|PublishDate: 2023-03-01|ExpiryDate: 0001-01-01|") + // Pages from gotmpl. + b.AssertFileContent("public/docs/p1/index.html", + "Single: p1:p1|", + "Path: /docs/p1|", + "<strong>Hello World</strong>", + "Params: param1v|", + "Len Resources: 7", + "RelPermalink: /mydata.yaml|Name: data1.yaml|Title: data1.yaml|Params: map[]|", + "RelPermalink: /mydata.yaml|Name: data2.yaml|Title: My data 2|Params: map[data2param1:data2Param1v]|", + "RelPermalink: /a/pixel.png|Name: featured.png|Title: My Featured Image|Params: map[data2param1:data2Param1v]|", + "RelPermalink: /docs/p1/sub/mytex2.txt|Name: sub/mytex2.txt|", + "RelPermalink: /docs/p1/sub/mymixcasetext2.txt|Name: sub/mymixcasetext2.txt|", + "RelPermalink: /mydata.yaml|Name: sub/data1.yaml|Title: Sub data|Params: map[]|", + "Featured Image: /a/pixel.png|featured.png|", + "Resized Featured Image: /a/pixel_hu8aa3346827e49d756ff4e630147c42b5_70_10x10_resize_box_3.png|10|", + // Resource from string + "RelPermalink: /docs/p1/mytext.txt|Name: textresource|Title: My Text Resource|Params: map[param1:param1v]|", + // Dates + "Dates: Date: 2023-03-01|Lastmod: 2023-03-01|PublishDate: 2023-03-01|ExpiryDate: 0001-01-01|", + ) + b.AssertFileContent("public/docs/p2/index.html", "Single: p2title|", "<b>Hello World!</b> No **markdown** here.") + b.AssertFileContent("public/docs/p3/index.html", "<strong>Hello World Default</strong>") +} + +func TestPagesFromGoTmplAsciidocAndSimilar(t *testing.T) { + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "rss", "sitemap"] +baseURL = "https://example.com" +[security] +[security.exec] +allow = ['asciidoctor', 'pandoc','rst2html', 'python'] +-- layouts/_default/single.html -- +|Content: {{ .Content }}|Title: {{ .Title }}|Path: {{ .Path }}| +-- content/docs/_content.gotmpl -- +{{ $.AddPage (dict "path" "asciidoc" "content" (dict "value" "Mark my words, #automation is essential#." "mediaType" "text/asciidoc" )) }} +{{ $.AddPage (dict "path" "pandoc" "content" (dict "value" "This ~~is deleted text.~~" "mediaType" "text/pandoc" )) }} +{{ $.AddPage (dict "path" "rst" "content" (dict "value" "This is *bold*." "mediaType" "text/rst" )) }} +{{ $.AddPage (dict "path" "org" "content" (dict "value" "the ability to use +strikethrough+ is a plus" "mediaType" "text/org" )) }} +{{ $.AddPage (dict "path" "nocontent" "title" "No Content" ) }} + + ` + + b := hugolib.Test(t, files) + + if asciidocext.Supports() { + b.AssertFileContent("public/docs/asciidoc/index.html", + "Mark my words, <mark>automation is essential</mark>", + "Path: /docs/asciidoc|", + ) + } + if pandoc.Supports() { + b.AssertFileContent("public/docs/pandoc/index.html", + "This <del>is deleted text.</del>", + "Path: /docs/pandoc|", + ) + } + + if rst.Supports() { + b.AssertFileContent("public/docs/rst/index.html", + "This is <em>bold</em>", + "Path: /docs/rst|", + ) + } + + b.AssertFileContent("public/docs/org/index.html", + "the ability to use <del>strikethrough</del> is a plus", + "Path: /docs/org|", + ) + + b.AssertFileContent("public/docs/nocontent/index.html", "|Content: |Title: No Content|Path: /docs/nocontent|") +} + +func TestPagesFromGoTmplAddPageErrors(t *testing.T) { + filesTemplate := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "rss", "sitemap"] +baseURL = "https://example.com" +-- content/docs/_content.gotmpl -- +{{ $.AddPage DICT }} +` + + t.Run("AddPage, missing Path", func(t *testing.T) { + files := strings.ReplaceAll(filesTemplate, "DICT", `(dict "kind" "page" "title" "p1")`) + b, err := hugolib.TestE(t, files) + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, "_content.gotmpl:1:4") + b.Assert(err.Error(), qt.Contains, "error calling AddPage: path not set") + }) + + t.Run("AddPage, path starting with slash", func(t *testing.T) { + files := strings.ReplaceAll(filesTemplate, "DICT", `(dict "kind" "page" "title" "p1" "path" "/foo")`) + b, err := hugolib.TestE(t, files) + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, `path "/foo" must not start with a /`) + }) + + t.Run("AddPage, lang set", func(t *testing.T) { + files := strings.ReplaceAll(filesTemplate, "DICT", `(dict "kind" "page" "path" "p1" "lang" "en")`) + b, err := hugolib.TestE(t, files) + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, "_content.gotmpl:1:4") + b.Assert(err.Error(), qt.Contains, "error calling AddPage: lang must not be set") + }) + + t.Run("Site methods not ready", func(t *testing.T) { + filesTemplate := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "rss", "sitemap"] +baseURL = "https://example.com" +-- content/docs/_content.gotmpl -- +{{ .Site.METHOD }} +` + + for _, method := range []string{"RegularPages", "Pages", "AllPages", "AllRegularPages", "Home", "Sections", "GetPage", "Menus", "MainSections", "Taxonomies"} { + t.Run(method, func(t *testing.T) { + files := strings.ReplaceAll(filesTemplate, "METHOD", method) + b, err := hugolib.TestE(t, files) + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, fmt.Sprintf("error calling %s: this method cannot be called before the site is fully initialized", method)) + }) + } + }) +} + +func TestPagesFromGoTmplAddResourceErrors(t *testing.T) { + filesTemplate := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "rss", "sitemap"] +baseURL = "https://example.com" +-- content/docs/_content.gotmpl -- +{{ $.AddResource DICT }} +` + + t.Run("missing Path", func(t *testing.T) { + files := strings.ReplaceAll(filesTemplate, "DICT", `(dict "name" "r1")`) + b, err := hugolib.TestE(t, files) + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, "error calling AddResource: path not set") + }) +} + +func TestPagesFromGoTmplEditGoTmpl(t *testing.T) { + t.Parallel() + b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic) + b.EditFileReplaceAll("content/docs/_content.gotmpl", `"title" "p2title"`, `"title" "p2titleedited"`).Build() + b.AssertFileContent("public/docs/p2/index.html", "Single: p2titleedited|") + b.AssertFileContent("public/docs/index.html", "p2titleedited") +} + +func TestPagesFromGoTmplEditDataResource(t *testing.T) { + t.Parallel() + b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic) + b.AssertRenderCountPage(7) + b.EditFileReplaceAll("assets/mydata.yaml", "p1: \"p1\"", "p1: \"p1edited\"").Build() + b.AssertFileContent("public/docs/p1/index.html", "Single: p1edited:p1|") + b.AssertFileContent("public/docs/index.html", "p1edited") + b.AssertRenderCountPage(3) +} + +func TestPagesFromGoTmplEditPartial(t *testing.T) { + t.Parallel() + b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic) + b.EditFileReplaceAll("layouts/partials/get-value.html", "p1", "p1edited").Build() + b.AssertFileContent("public/docs/p1/index.html", "Single: p1:p1edited|") + b.AssertFileContent("public/docs/index.html", "p1edited") +} + +func TestPagesFromGoTmplRemovePage(t *testing.T) { + t.Parallel() + b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic) + b.EditFileReplaceAll("content/docs/_content.gotmpl", `{{ $.AddPage (dict "kind" "page" "path" "p2" "title" "p2title" "dates" $dates "content" $contentHTML ) }}`, "").Build() + b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$") +} + +func TestPagesFromGoTmplDraftPage(t *testing.T) { + t.Parallel() + b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic) + b.EditFileReplaceAll("content/docs/_content.gotmpl", `"draft" false`, `"draft" true`).Build() + b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p4title:/docs/p4|pfile:/docs/pfile|$") +} + +func TestPagesFromGoTmplDraftFlagFromResource(t *testing.T) { + t.Parallel() + b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic) + b.EditFileReplaceAll("assets/mydata.yaml", `draft: false`, `draft: true`).Build() + b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p3title:/docs/p3|pfile:/docs/pfile|$") + b.EditFileReplaceAll("assets/mydata.yaml", `draft: true`, `draft: false`).Build() + b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$") +} + +func TestPagesFromGoTmplMovePage(t *testing.T) { + t.Parallel() + b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic) + b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$") + b.EditFileReplaceAll("content/docs/_content.gotmpl", `"path" "p2"`, `"path" "p2moved"`).Build() + b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2moved|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$") +} + +func TestPagesFromGoTmplRemoveGoTmpl(t *testing.T) { + t.Parallel() + b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic) + b.AssertFileContent("public/index.html", + "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$", + "Sections: Docs:/docs|", + ) + b.AssertFileContent("public/docs/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p2title:/docs/p2|p3title:/docs/p3|p4title:/docs/p4|pfile:/docs/pfile|$") + b.RemoveFiles("content/docs/_content.gotmpl").Build() + // One regular page left. + b.AssertFileContent("public/index.html", + "RegularPagesRecursive: pfile:/docs/pfile|$", + "Sections: Docs:/docs|", + ) + b.AssertFileContent("public/docs/index.html", "RegularPagesRecursive: pfile:/docs/pfile|$") +} + +func TestPagesFromGoTmplLanguagePerFile(t *testing.T) { + filesTemplate := ` +-- hugo.toml -- +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +[languages] +[languages.en] +weight = 1 +title = "Title" +[languages.fr] +weight = 2 +title = "Titre" +disabled = DISABLE +-- layouts/_default/single.html -- +Single: {{ .Title }}|{{ .Content }}| +-- content/docs/_content.gotmpl -- +{{ $.AddPage (dict "kind" "page" "path" "p1" "title" "Title" ) }} +-- content/docs/_content.fr.gotmpl -- +{{ $.AddPage (dict "kind" "page" "path" "p1" "title" "Titre" ) }} +` + + for _, disable := range []bool{false, true} { + t.Run(fmt.Sprintf("disable=%t", disable), func(t *testing.T) { + b := hugolib.Test(t, strings.ReplaceAll(filesTemplate, "DISABLE", fmt.Sprintf("%t", disable))) + b.AssertFileContent("public/en/docs/p1/index.html", "Single: Title||") + b.AssertFileExists("public/fr/docs/p1/index.html", !disable) + if !disable { + b.AssertFileContent("public/fr/docs/p1/index.html", "Single: Titre||") + } + }) + } +} + +func TestPagesFromGoTmplEnableAllLanguages(t *testing.T) { + t.Parallel() + + filesTemplate := ` +-- hugo.toml -- +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +[languages] +[languages.en] +weight = 1 +title = "Title" +[languages.fr] +title = "Titre" +weight = 2 +disabled = DISABLE +-- i18n/en.yaml -- +title: Title +-- i18n/fr.yaml -- +title: Titre +-- content/docs/_content.gotmpl -- +{{ .EnableAllLanguages }} +{{ $titleFromStore := .Store.Get "title" }} +{{ if not $titleFromStore }} + {{ $titleFromStore = "notfound"}} + {{ .Store.Set "title" site.Title }} +{{ end }} +{{ $title := printf "%s:%s:%s" site.Title (i18n "title") $titleFromStore }} +{{ $.AddPage (dict "kind" "page" "path" "p1" "title" $title ) }} +-- layouts/_default/single.html -- +Single: {{ .Title }}|{{ .Content }}| + +` + + for _, disable := range []bool{false, true} { + t.Run(fmt.Sprintf("disable=%t", disable), func(t *testing.T) { + b := hugolib.Test(t, strings.ReplaceAll(filesTemplate, "DISABLE", fmt.Sprintf("%t", disable))) + b.AssertFileExists("public/fr/docs/p1/index.html", !disable) + if !disable { + b.AssertFileContent("public/en/docs/p1/index.html", "Single: Title:Title:notfound||") + b.AssertFileContent("public/fr/docs/p1/index.html", "Single: Titre:Titre:Title||") + } + }) + } +} + +func TestPagesFromGoTmplMarkdownify(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "rss", "sitemap"] +baseURL = "https://example.com" +-- layouts/_default/single.html -- +|Content: {{ .Content }}|Title: {{ .Title }}|Path: {{ .Path }}| +-- content/docs/_content.gotmpl -- +{{ $content := "**Hello World**" | markdownify }} +{{ $.AddPage (dict "path" "p1" "content" (dict "value" $content "mediaType" "text/html" )) }} +` + + b, err := hugolib.TestE(t, files) + + // This currently fails. We should fix this, but that is not a trivial task, so do it later. + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, "error calling markdownify: this method cannot be called before the site is fully initialized") +} + +func TestPagesFromGoTmplResourceWithoutExtensionWithMediaTypeProvided(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "rss", "sitemap"] +baseURL = "https://example.com" +-- layouts/_default/single.html -- +|Content: {{ .Content }}|Title: {{ .Title }}|Path: {{ .Path }}| +{{ range .Resources }} +|RelPermalink: {{ .RelPermalink }}|Name: {{ .Name }}|Title: {{ .Title }}|Params: {{ .Params }}|MediaType: {{ .MediaType }}| +{{ end }} +-- content/docs/_content.gotmpl -- +{{ $.AddPage (dict "path" "p1" "content" (dict "value" "**Hello World**" "mediaType" "text/markdown" )) }} +{{ $.AddResource (dict "path" "p1/myresource" "content" (dict "value" "abcde" "mediaType" "text/plain" )) }} +` + + b := hugolib.Test(t, files) + + b.AssertFileContent("public/docs/p1/index.html", "RelPermalink: /docs/p1/myresource|Name: myresource|Title: myresource|Params: map[]|MediaType: text/plain|") +} + +func TestPagesFromGoTmplCascade(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "rss", "sitemap"] +baseURL = "https://example.com" +-- layouts/_default/single.html -- +|Content: {{ .Content }}|Title: {{ .Title }}|Path: {{ .Path }}|Params: {{ .Params }}| +-- content/_content.gotmpl -- +{{ $cascade := dict "params" (dict "cascadeparam1" "cascadeparam1value" ) }} +{{ $.AddPage (dict "path" "docs" "kind" "section" "cascade" $cascade ) }} +{{ $.AddPage (dict "path" "docs/p1" "content" (dict "value" "**Hello World**" "mediaType" "text/markdown" )) }} + +` + b := hugolib.Test(t, files) + + b.AssertFileContent("public/docs/p1/index.html", "|Path: /docs/p1|Params: map[cascadeparam1:cascadeparam1value") +} + +func TestPagesFromGoBuildOptions(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "rss", "sitemap"] +baseURL = "https://example.com" +-- layouts/_default/single.html -- +|Content: {{ .Content }}|Title: {{ .Title }}|Path: {{ .Path }}|Params: {{ .Params }}| +-- content/_content.gotmpl -- +{{ $.AddPage (dict "path" "docs/p1" "content" (dict "value" "**Hello World**" "mediaType" "text/markdown" )) }} +{{ $never := dict "list" "never" "publishResources" false "render" "never" }} +{{ $.AddPage (dict "path" "docs/p2" "content" (dict "value" "**Hello World**" "mediaType" "text/markdown" ) "build" $never ) }} + + +` + b := hugolib.Test(t, files) + + b.AssertFileExists("public/docs/p1/index.html", true) + b.AssertFileExists("public/docs/p2/index.html", false) +} diff --git a/hugolib/pagesfromdata/pagesfromgotmpl_test.go b/hugolib/pagesfromdata/pagesfromgotmpl_test.go new file mode 100644 index 000000000..c60b56dbf --- /dev/null +++ b/hugolib/pagesfromdata/pagesfromgotmpl_test.go @@ -0,0 +1,32 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pagesfromdata + +import "testing" + +func BenchmarkHash(b *testing.B) { + m := map[string]any{ + "foo": "bar", + "bar": "foo", + "stringSlice": []any{"a", "b", "c"}, + "intSlice": []any{1, 2, 3}, + "largeText": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. Sed sit amet ipsum mauris. Maecenas congue ligula ac quam viverra nec consectetur ante hendrerit.", + } + + bs := BuildState{} + + for i := 0; i < b.N; i++ { + bs.hash(m) + } +} diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index af4454a89..8a478c9df 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -321,7 +321,7 @@ func prepareShortcode( // Allow the caller to delay the rendering of the shortcode if needed. var fn shortcodeRenderFunc = func(ctx context.Context) ([]byte, bool, error) { - if p.m.pageConfig.IsGoldmark && sc.doMarkup { + if p.m.pageConfig.ContentMediaType.IsMarkdown() && sc.doMarkup { // Signal downwards that the content rendered will be // parsed and rendered by Goldmark. ctx = tpl.Context.IsInGoldmark.Set(ctx, true) @@ -449,7 +449,7 @@ func doRenderShortcode( // unchanged. // 2 If inner does not have a newline, strip the wrapping <p> block and // the newline. - switch p.m.pageConfig.Markup { + switch p.m.pageConfig.Content.Markup { case "", "markdown": if match, _ := regexp.MatchString(innerNewlineRegexp, inner); !match { cleaner, err := regexp.Compile(innerCleanupRegexp) diff --git a/hugolib/site.go b/hugolib/site.go index 280387838..d9103e737 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -62,6 +62,7 @@ import ( ) func (s *Site) Taxonomies() page.TaxonomyList { + s.checkReady() s.init.taxonomies.Do(context.Background()) return s.taxonomies } @@ -204,6 +205,7 @@ type siteRenderingContext struct { } func (s *Site) Menus() navigation.Menus { + s.checkReady() s.init.menus.Do(context.Background()) return s.menus } @@ -372,19 +374,33 @@ func (s *Site) watching() bool { type whatChanged struct { mu sync.Mutex - contentChanged bool - identitySet identity.Identities + needsPagesAssembly bool + identitySet identity.Identities } func (w *whatChanged) Add(ids ...identity.Identity) { w.mu.Lock() defer w.mu.Unlock() + if w.identitySet == nil { + w.identitySet = make(identity.Identities) + } + for _, id := range ids { w.identitySet[id] = true } } +func (w *whatChanged) Clear() { + w.mu.Lock() + defer w.mu.Unlock() + w.clear() +} + +func (w *whatChanged) clear() { + w.identitySet = identity.Identities{} +} + func (w *whatChanged) Changes() []identity.Identity { if w == nil || w.identitySet == nil { return nil @@ -392,6 +408,14 @@ func (w *whatChanged) Changes() []identity.Identity { return w.identitySet.AsSlice() } +func (w *whatChanged) Drain() []identity.Identity { + w.mu.Lock() + defer w.mu.Unlock() + ids := w.identitySet.AsSlice() + w.clear() + return ids +} + // RegisterMediaTypes will register the Site's media types in the mime // package, so it will behave correctly with Hugo's built-in server. func (s *Site) RegisterMediaTypes() { @@ -786,6 +810,7 @@ func (s *Site) errorCollator(results <-chan error, errs chan<- error) { // as possible for existing sites. Most sites will use {{ .Site.GetPage "section" "my/section" }}, // i.e. 2 arguments, so we test for that. func (s *Site) GetPage(ref ...string) (page.Page, error) { + s.checkReady() p, err := s.s.getPageForRefs(ref...) if p == nil { diff --git a/hugolib/site_new.go b/hugolib/site_new.go index 496889295..788b80a3f 100644 --- a/hugolib/site_new.go +++ b/hugolib/site_new.go @@ -32,6 +32,7 @@ import ( "github.com/gohugoio/hugo/config/allconfig" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugolib/doctree" + "github.com/gohugoio/hugo/hugolib/pagesfromdata" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/langs/i18n" @@ -51,7 +52,15 @@ import ( var _ page.Site = (*Site)(nil) +type siteState int + +const ( + siteStateInit siteState = iota + siteStateReady +) + type Site struct { + state siteState conf *allconfig.Config language *langs.Language languagei int @@ -166,7 +175,8 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { treeResources: doctree.New( treeConfig, ), - treeTaxonomyEntries: doctree.NewTreeShiftTree[*weightedContentNode](doctree.DimensionLanguage.Index(), len(confm.Languages)), + treeTaxonomyEntries: doctree.NewTreeShiftTree[*weightedContentNode](doctree.DimensionLanguage.Index(), len(confm.Languages)), + treePagesFromTemplateAdapters: doctree.NewTreeShiftTree[*pagesfromdata.PagesFromTemplate](doctree.DimensionLanguage.Index(), len(confm.Languages)), } pageTrees.createMutableTrees() @@ -415,6 +425,7 @@ func (s *Site) Current() page.Site { // MainSections returns the list of main sections. func (s *Site) MainSections() []string { + s.checkReady() return s.conf.C.MainSections } @@ -433,6 +444,7 @@ func (s *Site) BaseURL() string { // Deprecated: Use .Site.Lastmod instead. func (s *Site) LastChange() time.Time { + s.checkReady() hugo.Deprecate(".Site.LastChange", "Use .Site.Lastmod instead.", "v0.123.0") return s.lastmod } @@ -521,6 +533,7 @@ func (s *Site) ForEeachIdentityByName(name string, f func(identity.Identity) boo // Pages returns all pages. // This is for the current language only. func (s *Site) Pages() page.Pages { + s.checkReady() return s.pageMap.getPagesInSection( pageMapQueryPagesInSection{ pageMapQueryPagesBelowPath: pageMapQueryPagesBelowPath{ @@ -537,6 +550,7 @@ func (s *Site) Pages() page.Pages { // RegularPages returns all the regular pages. // This is for the current language only. func (s *Site) RegularPages() page.Pages { + s.checkReady() return s.pageMap.getPagesInSection( pageMapQueryPagesInSection{ pageMapQueryPagesBelowPath: pageMapQueryPagesBelowPath{ @@ -551,10 +565,18 @@ func (s *Site) RegularPages() page.Pages { // AllPages returns all pages for all sites. func (s *Site) AllPages() page.Pages { + s.checkReady() return s.h.Pages() } // AllRegularPages returns all regular pages for all sites. func (s *Site) AllRegularPages() page.Pages { + s.checkReady() return s.h.RegularPages() } + +func (s *Site) checkReady() { + if s.state != siteStateReady { + panic("this method cannot be called before the site is fully initialized") + } +} diff --git a/hugolib/site_sections.go b/hugolib/site_sections.go index 1ce091f59..03d662b9f 100644 --- a/hugolib/site_sections.go +++ b/hugolib/site_sections.go @@ -19,10 +19,12 @@ import ( // Sections returns the top level sections. func (s *Site) Sections() page.Pages { + s.checkReady() return s.Home().Sections() } // Home is a shortcut to the home page, equivalent to .Site.GetPage "home". func (s *Site) Home() page.Page { + s.checkReady() return s.s.home } |