diff options
61 files changed, 4495 insertions, 978 deletions
diff --git a/commands/hugobuilder.go b/commands/hugobuilder.go index 51493938d..dcc5c099b 100644 --- a/commands/hugobuilder.go +++ b/commands/hugobuilder.go @@ -920,7 +920,11 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher, changed := c.changeDetector.changed() if c.changeDetector != nil { - lrl.Logf("build changed %d files", len(changed)) + if len(changed) >= 10 { + lrl.Logf("build changed %d files", len(changed)) + } else { + lrl.Logf("build changed %d files: %q", len(changed), changed) + } if len(changed) == 0 { // Nothing has changed. return diff --git a/commands/server.go b/commands/server.go index c2fee68b2..c4a8ddd29 100644 --- a/commands/server.go +++ b/commands/server.go @@ -32,6 +32,7 @@ import ( "path" "path/filepath" "regexp" + "sort" "strconv" "strings" "sync" @@ -210,16 +211,17 @@ func (f *fileChangeDetector) changed() []string { } } - return f.filterIrrelevant(c) + return f.filterIrrelevantAndSort(c) } -func (f *fileChangeDetector) filterIrrelevant(in []string) []string { +func (f *fileChangeDetector) filterIrrelevantAndSort(in []string) []string { var filtered []string for _, v := range in { if !f.irrelevantRe.MatchString(v) { filtered = append(filtered, v) } } + sort.Strings(filtered) return filtered } diff --git a/common/herrors/errors.go b/common/herrors/errors.go index 40833e55c..c7ee90dd0 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -133,6 +133,21 @@ func IsNotExist(err error) bool { return false } +// IsExist returns true if the error is a file exists error. +// Unlike os.IsExist, this also considers wrapped errors. +func IsExist(err error) bool { + if os.IsExist(err) { + return true + } + + // os.IsExist does not consider wrapped errors. + if os.IsExist(errors.Unwrap(err)) { + return true + } + + return false +} + var nilPointerErrRe = regexp.MustCompile(`at <(.*)>: error calling (.*?): runtime error: invalid memory address or nil pointer dereference`) const deferredPrefix = "__hdeferred/" diff --git a/common/herrors/file_error.go b/common/herrors/file_error.go index 007a06b48..c8a48823e 100644 --- a/common/herrors/file_error.go +++ b/common/herrors/file_error.go @@ -384,7 +384,7 @@ func extractPosition(e error) (pos text.Position) { case godartsass.SassError: span := v.Span start := span.Start - filename, _ := paths.UrlToFilename(span.Url) + filename, _ := paths.UrlStringToFilename(span.Url) pos.Filename = filename pos.Offset = start.Offset pos.ColumnNumber = start.Column diff --git a/common/hreflect/helpers.go b/common/hreflect/helpers.go index 5113a3886..fc83165c7 100644 --- a/common/hreflect/helpers.go +++ b/common/hreflect/helpers.go @@ -223,6 +223,27 @@ func AsTime(v reflect.Value, loc *time.Location) (time.Time, bool) { return time.Time{}, false } +// ToSliceAny converts the given value to a slice of any if possible. +func ToSliceAny(v any) ([]any, bool) { + if v == nil { + return nil, false + } + switch vv := v.(type) { + case []any: + return vv, true + default: + vvv := reflect.ValueOf(v) + if vvv.Kind() == reflect.Slice { + out := make([]any, vvv.Len()) + for i := 0; i < vvv.Len(); i++ { + out[i] = vvv.Index(i).Interface() + } + return out, true + } + } + return nil, false +} + func CallMethodByName(cxt context.Context, name string, v reflect.Value) []reflect.Value { fn := v.MethodByName(name) var args []reflect.Value diff --git a/common/hreflect/helpers_test.go b/common/hreflect/helpers_test.go index 27b774337..119722261 100644 --- a/common/hreflect/helpers_test.go +++ b/common/hreflect/helpers_test.go @@ -50,6 +50,19 @@ func TestIsContextType(t *testing.T) { c.Assert(IsContextType(reflect.TypeOf(valueCtx)), qt.IsTrue) } +func TestToSliceAny(t *testing.T) { + c := qt.New(t) + + checkOK := func(in any, expected []any) { + out, ok := ToSliceAny(in) + c.Assert(ok, qt.Equals, true) + c.Assert(out, qt.DeepEquals, expected) + } + + checkOK([]any{1, 2, 3}, []any{1, 2, 3}) + checkOK([]int{1, 2, 3}, []any{1, 2, 3}) +} + func BenchmarkIsContextType(b *testing.B) { type k string b.Run("value", func(b *testing.B) { diff --git a/common/maps/cache.go b/common/maps/cache.go index cdc31a684..72c07e7ca 100644 --- a/common/maps/cache.go +++ b/common/maps/cache.go @@ -113,11 +113,14 @@ func (c *Cache[K, T]) set(key K, value T) { } // ForEeach calls the given function for each key/value pair in the cache. -func (c *Cache[K, T]) ForEeach(f func(K, T)) { +// If the function returns false, the iteration stops. +func (c *Cache[K, T]) ForEeach(f func(K, T) bool) { c.RLock() defer c.RUnlock() for k, v := range c.m { - f(k, v) + if !f(k, v) { + return + } } } diff --git a/common/paths/url.go b/common/paths/url.go index 75b4b644a..867f68baf 100644 --- a/common/paths/url.go +++ b/common/paths/url.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import ( "net/url" "path" "path/filepath" + "runtime" "strings" ) @@ -159,9 +160,77 @@ func Uglify(in string) string { return path.Clean(in) } +// URLEscape escapes unicode letters. +func URLEscape(uri string) string { + // escape unicode letters + u, err := url.Parse(uri) + if err != nil { + panic(err) + } + return u.String() +} + +// TrimExt trims the extension from a path.. +func TrimExt(in string) string { + return strings.TrimSuffix(in, path.Ext(in)) +} + +// From https://github.com/golang/go/blob/e0c76d95abfc1621259864adb3d101cf6f1f90fc/src/cmd/go/internal/web/url.go#L45 +func UrlFromFilename(filename string) (*url.URL, error) { + if !filepath.IsAbs(filename) { + return nil, fmt.Errorf("filepath must be absolute") + } + + // If filename has a Windows volume name, convert the volume to a host and prefix + // per https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/. + if vol := filepath.VolumeName(filename); vol != "" { + if strings.HasPrefix(vol, `\\`) { + filename = filepath.ToSlash(filename[2:]) + i := strings.IndexByte(filename, '/') + + if i < 0 { + // A degenerate case. + // \\host.example.com (without a share name) + // becomes + // file://host.example.com/ + return &url.URL{ + Scheme: "file", + Host: filename, + Path: "/", + }, nil + } + + // \\host.example.com\Share\path\to\file + // becomes + // file://host.example.com/Share/path/to/file + return &url.URL{ + Scheme: "file", + Host: filename[:i], + Path: filepath.ToSlash(filename[i:]), + }, nil + } + + // C:\path\to\file + // becomes + // file:///C:/path/to/file + return &url.URL{ + Scheme: "file", + Path: "/" + filepath.ToSlash(filename), + }, nil + } + + // /path/to/file + // becomes + // file:///path/to/file + return &url.URL{ + Scheme: "file", + Path: filepath.ToSlash(filename), + }, nil +} + // UrlToFilename converts the URL s to a filename. // If ParseRequestURI fails, the input is just converted to OS specific slashes and returned. -func UrlToFilename(s string) (string, bool) { +func UrlStringToFilename(s string) (string, bool) { u, err := url.ParseRequestURI(s) if err != nil { return filepath.FromSlash(s), false @@ -171,25 +240,34 @@ func UrlToFilename(s string) (string, bool) { if p == "" { p, _ = url.QueryUnescape(u.Opaque) - return filepath.FromSlash(p), true + return filepath.FromSlash(p), false + } + + if runtime.GOOS != "windows" { + return p, true + } + + if len(p) == 0 || p[0] != '/' { + return filepath.FromSlash(p), false } p = filepath.FromSlash(p) - if u.Host != "" { - // C:\data\file.txt - p = strings.ToUpper(u.Host) + ":" + p + if len(u.Host) == 1 { + // file://c/Users/... + return strings.ToUpper(u.Host) + ":" + p, true } - return p, true -} + if u.Host != "" && u.Host != "localhost" { + if filepath.VolumeName(u.Host) != "" { + return "", false + } + return `\\` + u.Host + p, true + } -// URLEscape escapes unicode letters. -func URLEscape(uri string) string { - // escape unicode letters - u, err := url.Parse(uri) - if err != nil { - panic(err) + if vol := filepath.VolumeName(p[1:]); vol == "" || strings.HasPrefix(vol, `\\`) { + return "", false } - return u.String() + + return p[1:], true } diff --git a/common/types/closer.go b/common/types/closer.go index 2844b1986..9f8875a8a 100644 --- a/common/types/closer.go +++ b/common/types/closer.go @@ -19,6 +19,13 @@ type Closer interface { Close() error } +// CloserFunc is a convenience type to create a Closer from a function. +type CloserFunc func() error + +func (f CloserFunc) Close() error { + return f() +} + type CloseAdder interface { Add(Closer) } diff --git a/config/allconfig/configlanguage.go b/config/allconfig/configlanguage.go index 38d2309ef..deec61449 100644 --- a/config/allconfig/configlanguage.go +++ b/config/allconfig/configlanguage.go @@ -137,11 +137,11 @@ func (c ConfigLanguage) Watching() bool { return c.m.Base.Internal.Watch } -func (c ConfigLanguage) NewIdentityManager(name string) identity.Manager { +func (c ConfigLanguage) NewIdentityManager(name string, opts ...identity.ManagerOption) identity.Manager { if !c.Watching() { return identity.NopManager } - return identity.NewManager(name) + return identity.NewManager(name, opts...) } func (c ConfigLanguage) ContentTypes() config.ContentTypesProvider { diff --git a/config/configProvider.go b/config/configProvider.go index ee6691cf1..5bda2c55a 100644 --- a/config/configProvider.go +++ b/config/configProvider.go @@ -58,7 +58,7 @@ type AllProvider interface { BuildDrafts() bool Running() bool Watching() bool - NewIdentityManager(name string) identity.Manager + NewIdentityManager(name string, opts ...identity.ManagerOption) identity.Manager FastRenderMode() bool PrintUnusedTemplates() bool EnableMissingTranslationPlaceholders() bool diff --git a/deps/deps.go b/deps/deps.go index 4389036cb..ca0f5ee3b 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "os" "path/filepath" "sort" "strings" @@ -47,6 +48,9 @@ type Deps struct { // The templates to use. This will usually implement the full tpl.TemplateManager. tmplHandlers *tpl.TemplateHandlers + // The template funcs. + TmplFuncMap map[string]any + // The file systems to use. Fs *hugofs.Fs `json:"-"` @@ -83,10 +87,13 @@ type Deps struct { Metrics metrics.Provider // BuildStartListeners will be notified before a build starts. - BuildStartListeners *Listeners + BuildStartListeners *Listeners[any] // BuildEndListeners will be notified after a build finishes. - BuildEndListeners *Listeners + BuildEndListeners *Listeners[any] + + // OnChangeListeners will be notified when something changes. + OnChangeListeners *Listeners[identity.Identity] // Resources that gets closed when the build is done or the server shuts down. BuildClosers *types.Closers @@ -154,17 +161,21 @@ func (d *Deps) Init() error { } if d.BuildStartListeners == nil { - d.BuildStartListeners = &Listeners{} + d.BuildStartListeners = &Listeners[any]{} } if d.BuildEndListeners == nil { - d.BuildEndListeners = &Listeners{} + d.BuildEndListeners = &Listeners[any]{} } if d.BuildClosers == nil { d.BuildClosers = &types.Closers{} } + if d.OnChangeListeners == nil { + d.OnChangeListeners = &Listeners[identity.Identity]{} + } + if d.Metrics == nil && d.Conf.TemplateMetrics() { d.Metrics = metrics.NewProvider(d.Conf.TemplateMetricsHints()) } @@ -268,6 +279,23 @@ func (d *Deps) Compile(prototype *Deps) error { return nil } +// MkdirTemp returns a temporary directory path that will be cleaned up on exit. +func (d Deps) MkdirTemp(pattern string) (string, error) { + filename, err := os.MkdirTemp("", pattern) + if err != nil { + return "", err + } + d.BuildClosers.Add( + types.CloserFunc( + func() error { + return os.RemoveAll(filename) + }, + ), + ) + + return filename, nil +} + type globalErrHandler struct { logger loggers.Logger @@ -306,15 +334,16 @@ func (e *globalErrHandler) StopErrorCollector() { } // Listeners represents an event listener. -type Listeners struct { +type Listeners[T any] struct { sync.Mutex // A list of funcs to be notified about an event. - listeners []func() + // If the return value is true, the listener will be removed. + listeners []func(...T) bool } // Add adds a function to a Listeners instance. -func (b *Listeners) Add(f func()) { +func (b *Listeners[T]) Add(f func(...T) bool) { if b == nil { return } @@ -324,12 +353,16 @@ func (b *Listeners) Add(f func()) { } // Notify executes all listener functions. -func (b *Listeners) Notify() { +func (b *Listeners[T]) Notify(vs ...T) { b.Lock() defer b.Unlock() + temp := b.listeners[:0] for _, notify := range b.listeners { - notify() + if !notify(vs...) { + temp = append(temp, notify) + } } + b.listeners = temp } // ResourceProvider is used to create and refresh, and clone resources needed. diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index a336c8489..eda6f769e 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -1754,6 +1754,11 @@ func (sa *sitePagesAssembler) assembleResources() error { mt = rs.rc.ContentMediaType } + var filename string + if rs.fi != nil { + filename = rs.fi.Meta().Filename + } + rd := resources.ResourceSourceDescriptor{ OpenReadSeekCloser: rs.opener, Path: rs.path, @@ -1762,6 +1767,7 @@ func (sa *sitePagesAssembler) assembleResources() error { TargetBasePaths: targetBasePaths, BasePathRelPermalink: targetPaths.SubResourceBaseLink, BasePathTargetPath: baseTarget, + SourceFilenameOrPath: filename, NameNormalized: relPath, NameOriginal: relPathOriginal, MediaType: mt, diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index a5186fd44..792d6a990 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -111,6 +111,10 @@ func (h *HugoSites) ShouldSkipFileChangeEvent(ev fsnotify.Event) bool { return h.skipRebuildForFilenames[ev.Name] } +func (h *HugoSites) Close() error { + return h.Deps.Close() +} + func (h *HugoSites) isRebuild() bool { return h.buildCounter.Load() > 0 } diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index dd548be51..5346e2e6b 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -520,8 +520,9 @@ func (s *Site) executeDeferredTemplates(de *deps.DeferredExecutions) error { }, }) - de.FilenamesWithPostPrefix.ForEeach(func(filename string, _ bool) { + de.FilenamesWithPostPrefix.ForEeach(func(filename string, _ bool) bool { g.Enqueue(filename) + return true }) return g.Wait() @@ -1058,6 +1059,8 @@ func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLo } } + h.Deps.OnChangeListeners.Notify(changed.Changes()...) + if err := h.resolveAndClearStateForIdentities(ctx, l, cacheBusterOr, changed.Drain()); err != nil { return err } diff --git a/hugolib/hugo_sites_build_errors_test.go b/hugolib/hugo_sites_build_errors_test.go index 71afe6767..1298d7f4f 100644 --- a/hugolib/hugo_sites_build_errors_test.go +++ b/hugolib/hugo_sites_build_errors_test.go @@ -554,8 +554,6 @@ toc line 3 toc line 4 - - ` t.Run("base template", func(t *testing.T) { @@ -569,7 +567,7 @@ toc line 4 ).BuildE() b.Assert(err, qt.IsNotNil) - b.Assert(err.Error(), qt.Contains, filepath.FromSlash(`render of "home" failed: "/layouts/baseof.html:4:6"`)) + b.Assert(err.Error(), qt.Contains, `baseof.html:4:6`) }) t.Run("index template", func(t *testing.T) { @@ -583,7 +581,7 @@ toc line 4 ).BuildE() b.Assert(err, qt.IsNotNil) - b.Assert(err.Error(), qt.Contains, filepath.FromSlash(`render of "home" failed: "/layouts/index.html:3:7"`)) + b.Assert(err.Error(), qt.Contains, `index.html:3:7"`) }) t.Run("partial from define", func(t *testing.T) { @@ -597,8 +595,7 @@ toc line 4 ).BuildE() b.Assert(err, qt.IsNotNil) - b.Assert(err.Error(), qt.Contains, filepath.FromSlash(`render of "home" failed: "/layouts/index.html:7:8": execute of template failed`)) - b.Assert(err.Error(), qt.Contains, `execute of template failed: template: partials/toc.html:2:8: executing "partials/toc.html"`) + b.Assert(err.Error(), qt.Contains, `toc.html:2:8"`) }) } diff --git a/hugolib/integrationtest_builder.go b/hugolib/integrationtest_builder.go index 637e5dee0..bc83d65b3 100644 --- a/hugolib/integrationtest_builder.go +++ b/hugolib/integrationtest_builder.go @@ -69,6 +69,13 @@ func TestOptDebug() TestOpt { } } +// TestOptInfo will enable info logging in integration tests. +func TestOptInfo() TestOpt { + return func(c *IntegrationTestConfig) { + c.LogLevel = logg.LevelInfo + } +} + // TestOptWarn will enable warn logging in integration tests. func TestOptWarn() TestOpt { return func(c *IntegrationTestConfig) { @@ -90,6 +97,13 @@ func TestOptWithNFDOnDarwin() TestOpt { } } +// TestOptWithOSFs enables the real file system. +func TestOptWithOSFs() TestOpt { + return func(c *IntegrationTestConfig) { + c.NeedsOsFS = true + } +} + // TestOptWithWorkingDir allows setting any config optiona as a function al option. func TestOptWithConfig(fn func(c *IntegrationTestConfig)) TestOpt { return func(c *IntegrationTestConfig) { @@ -284,8 +298,9 @@ func (s *IntegrationTestBuilder) negate(match string) (string, bool) { func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...string) { s.Helper() content := strings.TrimSpace(s.FileContent(filename)) + for _, m := range matches { - cm := qt.Commentf("File: %s Match %s", filename, m) + cm := qt.Commentf("File: %s Match %s\nContent:\n%s", filename, m, content) lines := strings.Split(m, "\n") for _, match := range lines { match = strings.TrimSpace(match) @@ -313,7 +328,8 @@ func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches s.Helper() content := s.FileContent(filename) for _, m := range matches { - s.Assert(content, qt.Contains, m, qt.Commentf(m)) + cm := qt.Commentf("File: %s Match %s\nContent:\n%s", filename, m, content) + s.Assert(content, qt.Contains, m, cm) } } @@ -450,6 +466,11 @@ func (s *IntegrationTestBuilder) Build() *IntegrationTestBuilder { return s } +func (s *IntegrationTestBuilder) Close() { + s.Helper() + s.Assert(s.H.Close(), qt.IsNil) +} + func (s *IntegrationTestBuilder) LogString() string { return s.lastBuildLog } diff --git a/hugolib/page.go b/hugolib/page.go index e4c841966..83f0c6e25 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -143,6 +143,10 @@ func (p *pageState) GetDependencyManagerForScope(scope int) identity.Manager { } } +func (p *pageState) GetDependencyManagerForScopesAll() []identity.Manager { + return []identity.Manager{p.dependencyManager, p.dependencyManagerOutput} +} + func (p *pageState) Key() string { return "page-" + strconv.FormatUint(p.pid, 10) } diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go index 96c2c0f96..f175895c4 100644 --- a/hugolib/pages_capture.go +++ b/hugolib/pages_capture.go @@ -143,13 +143,29 @@ func (c *pagesCollector) Collect() (collectErr error) { s.pageMap.cfg.isRebuild = true } + var hasStructuralChange bool + for _, id := range c.ids { + if id.isStructuralChange() { + hasStructuralChange = true + break + } + } + for _, id := range c.ids { if id.p.IsLeafBundle() { collectErr = c.collectDir( id.p, false, func(fim hugofs.FileMetaInfo) bool { - return true + if hasStructuralChange { + return true + } + fimp := fim.Meta().PathInfo + if fimp == nil { + return true + } + + return fimp.Path() == id.p.Path() }, ) } else if id.p.IsBranchBundle() { diff --git a/hugolib/pagesfromdata/pagesfromgotmpl.go b/hugolib/pagesfromdata/pagesfromgotmpl.go index fd7213bd9..2ee273718 100644 --- a/hugolib/pagesfromdata/pagesfromgotmpl.go +++ b/hugolib/pagesfromdata/pagesfromgotmpl.go @@ -245,10 +245,11 @@ func (b *BuildState) resolveDeletedPaths() { return } var paths []string - b.sourceInfosPrevious.ForEeach(func(k string, _ *sourceInfo) { + b.sourceInfosPrevious.ForEeach(func(k string, _ *sourceInfo) bool { if _, found := b.sourceInfosCurrent.Get(k); !found { paths = append(paths, k) } + return true }) b.DeletedPaths = paths @@ -287,6 +288,10 @@ func (p *PagesFromTemplate) GetDependencyManagerForScope(scope int) identity.Man return p.DependencyManager } +func (p *PagesFromTemplate) GetDependencyManagerForScopesAll() []identity.Manager { + return []identity.Manager{p.DependencyManager} +} + func (p *PagesFromTemplate) Execute(ctx context.Context) (BuildInfo, error) { defer func() { p.buildState.PrepareNextBuild() diff --git a/hugolib/rebuild_test.go b/hugolib/rebuild_test.go index ff1f8267e..42e1b8ed5 100644 --- a/hugolib/rebuild_test.go +++ b/hugolib/rebuild_test.go @@ -98,6 +98,18 @@ My Other Text: {{ $r.Content }}|{{ $r.Permalink }}| ` +func TestRebuildEditLeafBundleHeaderOnly(t *testing.T) { + b := TestRunning(t, rebuildFilesSimple) + b.AssertFileContent("public/mysection/mysectionbundle/index.html", + "My Section Bundle Content Content.") + + b.EditFileReplaceAll("content/mysection/mysectionbundle/index.md", "My Section Bundle Content.", "My Section Bundle Content Edited.").Build() + b.AssertFileContent("public/mysection/mysectionbundle/index.html", + "My Section Bundle Content Edited.") + b.AssertRenderCountPage(2) // home (rss) + bundle. + b.AssertRenderCountContent(1) +} + func TestRebuildEditTextFileInLeafBundle(t *testing.T) { b := TestRunning(t, rebuildFilesSimple) b.AssertFileContent("public/mysection/mysectionbundle/index.html", @@ -962,7 +974,7 @@ Single. {{ partial "head.html" . }}$ RelPermalink: {{ $js.RelPermalink }}| ` - b := TestRunning(t, files) + b := TestRunning(t, files, TestOptOsFs()) b.AssertFileContent("public/p1/index.html", "/js/main.712a50b59d0f0dedb4e3606eaa3860b1f1a5305f6c42da30a2985e473ba314eb.js") b.AssertFileContent("public/index.html", "/js/main.712a50b59d0f0dedb4e3606eaa3860b1f1a5305f6c42da30a2985e473ba314eb.js") @@ -998,7 +1010,7 @@ Base. {{ partial "common/head.html" . }}$ RelPermalink: {{ $js.RelPermalink }}| ` - b := TestRunning(t, files) + b := TestRunning(t, files, TestOptOsFs()) b.AssertFileContent("public/index.html", "/js/main.712a50b59d0f0dedb4e3606eaa3860b1f1a5305f6c42da30a2985e473ba314eb.js") diff --git a/hugolib/site.go b/hugolib/site.go index f6dc89a77..7ec70eb6c 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -1493,7 +1493,11 @@ func (s *Site) renderForTemplate(ctx context.Context, name, outputFormat string, } if err = s.Tmpl().ExecuteWithContext(ctx, templ, w, d); err != nil { - return fmt.Errorf("render of %q failed: %w", name, err) + filename := name + if p, ok := d.(*pageState); ok { + filename = p.String() + } + return fmt.Errorf("render of %q failed: %w", filename, err) } return } diff --git a/identity/identity.go b/identity/identity.go index d106eb1fc..e38a8f0f2 100644 --- a/identity/identity.go +++ b/identity/identity.go @@ -82,9 +82,8 @@ func FirstIdentity(v any) Identity { var result Identity = Anonymous WalkIdentitiesShallow(v, func(level int, id Identity) bool { result = id - return true + return result != Anonymous }) - return result } @@ -146,6 +145,7 @@ func (d DependencyManagerProviderFunc) GetDependencyManager() Manager { // DependencyManagerScopedProvider provides a manager for dependencies with a given scope. type DependencyManagerScopedProvider interface { GetDependencyManagerForScope(scope int) Manager + GetDependencyManagerForScopesAll() []Manager } // ForEeachIdentityProvider provides a way iterate over identities. @@ -308,11 +308,13 @@ type identityManager struct { func (im *identityManager) AddIdentity(ids ...Identity) { im.mu.Lock() + defer im.mu.Unlock() for _, id := range ids { if id == nil || id == Anonymous { continue } + if _, found := im.ids[id]; !found { if im.onAddIdentity != nil { im.onAddIdentity(id) @@ -320,7 +322,6 @@ func (im *identityManager) AddIdentity(ids ...Identity) { im.ids[id] = true } } - im.mu.Unlock() } func (im *identityManager) AddIdentityForEach(ids ...ForEeachIdentityProvider) { @@ -355,6 +356,10 @@ func (im *identityManager) GetDependencyManagerForScope(int) Manager { return im } +func (im *identityManager) GetDependencyManagerForScopesAll() []Manager { + return []Manager{im} +} + func (im *identityManager) String() string { return fmt.Sprintf("IdentityManager(%s)", im.name) } diff --git a/internal/js/esbuild/batch-esm-runner.gotmpl b/internal/js/esbuild/batch-esm-runner.gotmpl new file mode 100644 index 000000000..3193b4c30 --- /dev/null +++ b/internal/js/esbuild/batch-esm-runner.gotmpl @@ -0,0 +1,20 @@ +{{ range $i, $e := .Scripts -}} + {{ if eq .Export "*" }} + {{- printf "import %s as Script%d from %q;" .Export $i .Import -}} + {{ else -}} + {{- printf "import { %s as Script%d } from %q;" .Export $i .Import -}} + {{ end -}} +{{ end -}} +{{ range $i, $e := .Runners }} + {{- printf "import { %s as Run%d } from %q;" .Export $i .Import -}} +{{ end -}} +{{ if .Runners -}} + let group = { id: "{{ $.ID }}", scripts: [] } + {{ range $i, $e := .Scripts -}} + group.scripts.push({{ .RunnerJSON $i }}); + {{ end -}} + {{ range $i, $e := .Runners -}} + {{ $id := printf "Run%d" $i }} + {{ $id }}(group); + {{ end -}} +{{ end -}} diff --git a/internal/js/esbuild/batch.go b/internal/js/esbuild/batch.go new file mode 100644 index 000000000..43e2da444 --- /dev/null +++ b/internal/js/esbuild/batch.go @@ -0,0 +1,1437 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package esbuild provides functions for building JavaScript resources. +package esbuild + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "fmt" + "path" + "path/filepath" + "reflect" + "sort" + "strings" + "sync" + "sync/atomic" + + "github.com/evanw/esbuild/pkg/api" + "github.com/gohugoio/hugo/cache/dynacache" + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/lazy" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/resources/resource_factories/create" + "github.com/gohugoio/hugo/tpl" + "github.com/mitchellh/mapstructure" + "github.com/spf13/afero" + "github.com/spf13/cast" +) + +var _ Batcher = (*batcher)(nil) + +const ( + NsBatch = "_hugo-js-batch" + + propsKeyImportContext = "importContext" + propsResoure = "resource" +) + +//go:embed batch-esm-runner.gotmpl +var runnerTemplateStr string + +var _ BatchPackage = (*Package)(nil) + +var _ buildToucher = (*optsHolder[scriptOptions])(nil) + +var ( + _ buildToucher = (*scriptGroup)(nil) + _ isBuiltOrTouchedProvider = (*scriptGroup)(nil) +) + +func NewBatcherClient(deps *deps.Deps) (*BatcherClient, error) { + c := &BatcherClient{ + d: deps, + buildClient: NewBuildClient(deps.BaseFs.Assets, deps.ResourceSpec), + createClient: create.New(deps.ResourceSpec), + bundlesCache: maps.NewCache[string, BatchPackage](), + } + + deps.BuildEndListeners.Add(func(...any) bool { + c.bundlesCache.Reset() + return false + }) + + return c, nil +} + +func (o optionsMap[K, C]) ByKey() optionsGetSetters[K, C] { + var values []optionsGetSetter[K, C] + for _, v := range o { + values = append(values, v) + } + + sort.Slice(values, func(i, j int) bool { + return values[i].Key().String() < values[j].Key().String() + }) + + return values +} + +func (o *opts[K, C]) Compiled() C { + o.h.checkCompileErr() + return o.h.compiled +} + +func (os optionsGetSetters[K, C]) Filter(predicate func(K) bool) optionsGetSetters[K, C] { + var a optionsGetSetters[K, C] + for _, v := range os { + if predicate(v.Key()) { + a = append(a, v) + } + } + return a +} + +func (o *optsHolder[C]) IdentifierBase() string { + return o.optionsID +} + +func (o *opts[K, C]) Key() K { + return o.key +} + +func (o *opts[K, C]) Reset() { + mu := o.once.ResetWithLock() + defer mu.Unlock() + o.h.resetCounter++ +} + +func (o *opts[K, C]) Get(id uint32) OptionsSetter { + var b *optsHolder[C] + o.once.Do(func() { + b = o.h + b.setBuilt(id) + }) + return b +} + +func (o *opts[K, C]) GetIdentity() identity.Identity { + return o.h +} + +func (o *optsHolder[C]) SetOptions(m map[string]any) string { + o.optsSetCounter++ + o.optsPrev = o.optsCurr + o.optsCurr = m + o.compiledPrev = o.compiled + o.compiled, o.compileErr = o.compiled.compileOptions(m, o.defaults) + o.checkCompileErr() + return "" +} + +// ValidateBatchID validates the given ID according to some very +func ValidateBatchID(id string, isTopLevel bool) error { + if id == "" { + return fmt.Errorf("id must be set") + } + // No Windows slashes. + if strings.Contains(id, "\\") { + return fmt.Errorf("id must not contain backslashes") + } + + // Allow forward slashes in top level IDs only. + if !isTopLevel && strings.Contains(id, "/") { + return fmt.Errorf("id must not contain forward slashes") + } + + return nil +} + +func newIsBuiltOrTouched() isBuiltOrTouched { + return isBuiltOrTouched{ + built: make(buildIDs), + touched: make(buildIDs), + } +} + +func newOpts[K any, C optionsCompiler[C]](key K, optionsID string, defaults defaultOptionValues) *opts[K, C] { + return &opts[K, C]{ + key: key, + h: &optsHolder[C]{ + optionsID: optionsID, + defaults: defaults, + isBuiltOrTouched: newIsBuiltOrTouched(), + }, + } +} + +// BatchPackage holds a group of JavaScript resources. +type BatchPackage interface { + Groups() map[string]resource.Resources +} + +// Batcher is used to build JavaScript packages. +type Batcher interface { + Build(context.Context) (BatchPackage, error) + Config(ctx context.Context) OptionsSetter + Group(ctx context.Context, id string) BatcherGroup +} + +// BatcherClient is a client for building JavaScript packages. +type BatcherClient struct { + d *deps.Deps + + once sync.Once + runnerTemplate tpl.Template + + createClient *create.Client + buildClient *BuildClient + + bundlesCache *maps.Cache[string, BatchPackage] +} + +// New creates a new Batcher with the given ID. +// This will be typically created once and reused across rebuilds. +func (c *BatcherClient) New(id string) (Batcher, error) { + var initErr error + c.once.Do(func() { + // We should fix the initialization order here (or use the Go template package directly), but we need to wait + // for the Hugo templates to be ready. + tmpl, err := c.d.TextTmpl().Parse("batch-esm-runner", runnerTemplateStr) + if err != nil { + initErr = err + return + } + c.runnerTemplate = tmpl + }) + + if initErr != nil { + return nil, initErr + } + + dependencyManager := c.d.Conf.NewIdentityManager("jsbatch_" + id) + configID := "config_" + id + + b := &batcher{ + id: id, + scriptGroups: make(map[string]*scriptGroup), + dependencyManager: dependencyManager, + client: c, + configOptions: newOpts[scriptID, configOptions]( + scriptID(configID), + configID, + defaultOptionValues{}, + ), + } + + c.d.BuildEndListeners.Add(func(...any) bool { + b.reset() + return false + }) + + idFinder := identity.NewFinder(identity.FinderConfig{}) + + c.d.OnChangeListeners.Add(func(ids ...identity.Identity) bool { + for _, id := range ids { + if r := idFinder.Contains(id, b.dependencyManager, 50); r > 0 { + b.staleVersion.Add(1) + return false + } + + sp, ok := id.(identity.DependencyManagerScopedProvider) + if !ok { + continue + } + idms := sp.GetDependencyManagerForScopesAll() + + for _, g := range b.scriptGroups { + g.forEachIdentity(func(id2 identity.Identity) bool { + bt, ok := id2.(buildToucher) + if !ok { + return false + } + for _, id3 := range idms { + // This handles the removal of the only source for a script group (e.g. all shortcodes in a contnt page). + // Note the very shallow search. + if r := idFinder.Contains(id2, id3, 0); r > 0 { + bt.setTouched(b.buildCount) + return false + } + } + return false + }) + } + } + + return false + }) + + return b, nil +} + +func (c *BatcherClient) buildBatchGroup(ctx context.Context, t *batchGroupTemplateContext) (resource.Resource, string, error) { + var buf bytes.Buffer + + if err := c.d.Tmpl().ExecuteWithContext(ctx, c.runnerTemplate, &buf, t); err != nil { + return nil, "", err + } + + s := paths.AddLeadingSlash(t.keyPath + ".js") + r, err := c.createClient.FromString(s, buf.String()) + if err != nil { + return nil, "", err + } + + return r, s, nil +} + +// BatcherGroup is a group of scripts and instances. +type BatcherGroup interface { + Instance(sid, iid string) OptionsSetter + Runner(id string) OptionsSetter + Script(id string) OptionsSetter +} + +// OptionsSetter is used to set options for a batch, script or instance. +type OptionsSetter interface { + SetOptions(map[string]any) string +} + +// Package holds a group of JavaScript resources. +type Package struct { + id string + b *batcher + + groups map[string]resource.Resources +} + +func (p *Package) Groups() map[string]resource.Resources { + return p.groups +} + +type batchGroupTemplateContext struct { + keyPath string + ID string + Runners []scriptRunnerTemplateContext + Scripts []scriptBatchTemplateContext +} + +type batcher struct { + mu sync.Mutex + id string + buildCount uint32 + staleVersion atomic.Uint32 + scriptGroups scriptGroups + + client *BatcherClient + dependencyManager identity.Manager + + configOptions optionsGetSetter[scriptID, configOptions] + + // The last successfully built package. + // If this is non-nil and not stale, we can reuse it (e.g. on server rebuilds) + prevBuild *Package +} + +// Build builds the batch if not already built or if it's stale. +func (b *batcher) Build(ctx context.Context) (BatchPackage, error) { + key := dynacache.CleanKey(b.id + ".js") + p, err := b.client.bundlesCache.GetOrCreate(key, func() (BatchPackage, error) { + return b.build(ctx) + }) + if err != nil { + return nil, fmt.Errorf("failed to build JS batch %q: %w", b.id, err) + } + return p, nil +} + +func (b *batcher) Config(ctx context.Context) OptionsSetter { + return b.configOptions.Get(b.buildCount) +} + +func (b *batcher) Group(ctx context.Context, id string) BatcherGroup { + if err := ValidateBatchID(id, false); err != nil { + panic(err) + } + + b.mu.Lock() + defer b.mu.Unlock() + + group, found := b.scriptGroups[id] + if !found { + idm := b.client.d.Conf.NewIdentityManager("jsbatch_" + id) + b.dependencyManager.AddIdentity(idm) + + group = &scriptGroup{ + id: id, b: b, + isBuiltOrTouched: newIsBuiltOrTouched(), + dependencyManager: idm, + scriptsOptions: make(optionsMap[scriptID, scriptOptions]), + instancesOptions: make(optionsMap[instanceID, paramsOptions]), + runnersOptions: make(optionsMap[scriptID, scriptOptions]), + } + b.scriptGroups[id] = group + } + + group.setBuilt(b.buildCount) + + return group +} + +func (b *batcher) isStale() bool { + if b.staleVersion.Load() > 0 { + return true + } + + if b.removeNotSet() { + return true + } + + if b.configOptions.isStale() { + return true + } + + for _, v := range b.scriptGroups { + if v.isStale() { + return true + } + } + + return false +} + +func (b *batcher) build(ctx context.Context) (BatchPackage, error) { + b.mu.Lock() + defer b.mu.Unlock() + defer func() { + b.staleVersion.Store(0) + b.buildCount++ + }() + + if b.prevBuild != nil { + if !b.isStale() { + return b.prevBuild, nil + } + } + + p, err := b.doBuild(ctx) + if err != nil { + return nil, err + } + + b.prevBuild = p + + return p, nil +} + +func (b *batcher) doBuild(ctx context.Context) (*Package, error) { + type importContext struct { + name string + resourceGetter resource.ResourceGetter + scriptOptions scriptOptions + dm identity.Manager + } + + state := struct { + importResource *maps.Cache[string, resource.Resource] + resultResource *maps.Cache[string, resource.Resource] + importerImportContext *maps.Cache[string, importContext] + pathGroup *maps.Cache[string, string] + }{ + importResource: maps.NewCache[string, resource.Resource](), + resultResource: maps.NewCache[string, resource.Resource](), + importerImportContext: maps.NewCache[string, importContext](), + pathGroup: maps.NewCache[string, string](), + } + + // Entry points passed to ESBuid. + var entryPoints []string + addResource := func(group, pth string, r resource.Resource, isResult bool) { + state.pathGroup.Set(paths.TrimExt(pth), group) + state.importResource.Set(pth, r) + if isResult { + state.resultResource.Set(pth, r) + } + entryPoints = append(entryPoints, pth) + } + + for k, v := range b.scriptGroups { + keyPath := k + var runners []scriptRunnerTemplateContext + for _, vv := range v.runnersOptions.ByKey() { + runnerKeyPath := keyPath + "_" + vv.Key().String() + runnerImpPath := paths.AddLeadingSlash(runnerKeyPath + "_runner" + vv.Compiled().Resource.MediaType().FirstSuffix.FullSuffix) + runners = append(runners, scriptRunnerTemplateContext{opts: vv, Import: runnerImpPath}) + addResource(k, runnerImpPath, vv.Compiled().Resource, false) + } + + t := &batchGroupTemplateContext{ + keyPath: keyPath, + ID: v.id, + Runners: runners, + } + + instances := v.instancesOptions.ByKey() + + for _, vv := range v.scriptsOptions.ByKey() { + keyPath := keyPath + "_" + vv.Key().String() + opts := vv.Compiled() + impPath := path.Join(PrefixHugoVirtual, opts.Dir(), keyPath+opts.Resource.MediaType().FirstSuffix.FullSuffix) + impCtx := opts.ImportContext + + state.importerImportContext.Set(impPath, importContext{ + name: keyPath, + resourceGetter: impCtx, + scriptOptions: opts, + dm: v.dependencyManager, + }) + + bt := scriptBatchTemplateContext{ + opts: vv, + Import: impPath, + } + state.importResource.Set(bt.Import, vv.Compiled().Resource) + predicate := func(k instanceID) bool { + return k.scriptID == vv.Key() + } + for _, vvv := range instances.Filter(predicate) { + bt.Instances = append(bt.Instances, scriptInstanceBatchTemplateContext{opts: vvv}) + } + + t.Scripts = append(t.Scripts, bt) + } + + r, s, err := b.client.buildBatchGroup(ctx, t) + if err != nil { + return nil, fmt.Errorf("failed to build JS batch: %w", err) + } + + state.importerImportContext.Set(s, importContext{ + name: s, + resourceGetter: nil, + dm: v.dependencyManager, + }) + + addResource(v.id, s, r, true) + } + + mediaTypes := b.client.d.ResourceSpec.MediaTypes() + + externalOptions := b.configOptions.Compiled().Options + if externalOptions.Format == "" { + externalOptions.Format = "esm" + } + if externalOptions.Format != "esm" { + return nil, fmt.Errorf("only esm format is currently supported") + } + + jsOpts := Options{ + ExternalOptions: externalOptions, + InternalOptions: InternalOptions{ + DependencyManager: b.dependencyManager, + Splitting: true, + ImportOnResolveFunc: func(imp string, args api.OnResolveArgs) string { + var importContextPath string + if args.Kind == api.ResolveEntryPoint { + importContextPath = args.Path + } else { + importContextPath = args.Importer + } + importContext, importContextFound := state.importerImportContext.Get(importContextPath) + + // We want to track the dependencies closest to where they're used. + dm := b.dependencyManager + if importContextFound { + dm = importContext.dm + } + + if r, found := state.importResource.Get(imp); found { + dm.AddIdentity(identity.FirstIdentity(r)) + return imp + } + + if importContext.resourceGetter != nil { + resolved := ResolveResource(imp, importContext.resourceGetter) + if resolved != nil { + resolvePath := resources.InternalResourceTargetPath(resolved) + dm.AddIdentity(identity.FirstIdentity(resolved)) + imp := PrefixHugoVirtual + resolvePath + state.importResource.Set(imp, resolved) + state.importerImportContext.Set(imp, importContext) + return imp + + } + } + return "" + }, + ImportOnLoadFunc: func(args api.OnLoadArgs) string { + imp := args.Path + + if r, found := state.importResource.Get(imp); found { + content, err := r.(resource.ContentProvider).Content(ctx) + if err != nil { + panic(err) + } + return cast.ToString(content) + } + return "" + }, + ImportParamsOnLoadFunc: func(args api.OnLoadArgs) json.RawMessage { + if importContext, found := state.importerImportContext.Get(args.Path); found { + if !importContext.scriptOptions.IsZero() { + return importContext.scriptOptions.Params + } + } + return nil + }, + ErrorMessageResolveFunc: func(args api.Message) *ErrorMessageResolved { + if loc := args.Location; loc != nil { + path := strings.TrimPrefix(loc.File, NsHugoImportResolveFunc+":") + if r, found := state.importResource.Get(path); found { + sourcePath := resources.InternalResourceSourcePathBestEffort(r) + + var contentr hugio.ReadSeekCloser + if cp, ok := r.(hugio.ReadSeekCloserProvider); ok { + contentr, _ = cp.ReadSeekCloser() + } + return &ErrorMessageResolved{ + Content: contentr, + Path: sourcePath, + Message: args.Text, + } + + } + + } + return nil + }, + ResolveSourceMapSource: func(s string) string { + if r, found := state.importResource.Get(s); found { + if ss := resources.InternalResourceSourcePath(r); ss != "" { + return ss + } + return PrefixHugoMemory + s + } + return "" + }, + EntryPoints: entryPoints, + }, + } + + result, err := b.client.buildClient.Build(jsOpts) + if err != nil { + return nil, fmt.Errorf("failed to build JS bundle: %w", err) + } + + groups := make(map[string]resource.Resources) + + createAndAddResource := func(targetPath, group string, o api.OutputFile, mt media.Type) error { + var sourceFilename string + if r, found := state.importResource.Get(targetPath); found { + sourceFilename = resources.InternalResourceSourcePathBestEffort(r) + } + targetPath = path.Join(b.id, targetPath) + + rd := resources.ResourceSourceDescriptor{ + LazyPublish: true, + OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { + return hugio.NewReadSeekerNoOpCloserFromBytes(o.Contents), nil + }, + MediaType: mt, + TargetPath: targetPath, + SourceFilenameOrPath: sourceFilename, + } + r, err := b.client.d.ResourceSpec.NewResource(rd) + if err != nil { + return err + } + + groups[group] = append(groups[group], r) + + return nil + } + + outDir := b.client.d.AbsPublishDir + + createAndAddResources := func(o api.OutputFile) (bool, error) { + p := paths.ToSlashPreserveLeading(strings.TrimPrefix(o.Path, outDir)) + ext := path.Ext(p) + mt, _, found := mediaTypes.GetBySuffix(ext) + if !found { + return false, nil + } + + group, found := state.pathGroup.Get(paths.TrimExt(p)) + + if !found { + return false, nil + } + + if err := createAndAddResource(p, group, o, mt); err != nil { + return false, err + } + + return true, nil + } + + for _, o := range result.OutputFiles { + handled, err := createAndAddResources(o) + if err != nil { + return nil, err + } + + if !handled { + // Copy to destination. + p := strings.TrimPrefix(o.Path, outDir) + targetFilename := filepath.Join(b.id, p) + fs := b.client.d.BaseFs.PublishFs + if err := fs.MkdirAll(filepath.Dir(targetFilename), 0o777); err != nil { + return nil, fmt.Errorf("failed to create dir %q: %w", targetFilename, err) + } + + if err := afero.WriteFile(fs, targetFilename, o.Contents, 0o666); err != nil { + return nil, fmt.Errorf("failed to write to %q: %w", targetFilename, err) + } + } + } + + p := &Package{ + id: path.Join(NsBatch, b.id), + b: b, + groups: groups, + } + + return p, nil +} + +func (b *batcher) removeNotSet() bool { + // We already have the lock. + var removed bool + currentBuildID := b.buildCount + for k, v := range b.scriptGroups { + if !v.isBuilt(currentBuildID) && v.isTouched(currentBuildID) { + // Remove entire group. + removed = true + delete(b.scriptGroups, k) + continue + } + if v.removeTouchedButNotSet() { + removed = true + } + if v.removeNotSet() { + removed = true + } + } + + return removed +} + +func (b *batcher) reset() { + b.mu.Lock() + defer b.mu.Unlock() + b.configOptions.Reset() + for _, v := range b.scriptGroups { + v.Reset() + } +} + +type buildIDs map[uint32]bool + +func (b buildIDs) Has(buildID uint32) bool { + return b[buildID] +} + +func (b buildIDs) Set(buildID uint32) { + b[buildID] = true +} + +type buildToucher interface { + setTouched(buildID uint32) +} + +type configOptions struct { + Options ExternalOptions +} + +func (s configOptions) isStaleCompiled(prev configOptions) bool { + return false +} + +func (s configOptions) compileOptions(m map[string]any, defaults defaultOptionValues) (configOptions, error) { + config, err := DecodeExternalOptions(m) + if err != nil { + return configOptions{}, err + } + + return configOptions{ + Options: config, + }, nil +} + +type defaultOptionValues struct { + defaultExport string +} + +type instanceID struct { + scriptID scriptID + instanceID string +} + +func (i instanceID) String() string { + return i.scriptID.String() + "_" + i.instanceID +} + +type isBuiltOrTouched struct { + built buildIDs + touched buildIDs +} + +func (i isBuiltOrTouched) setBuilt(id uint32) { + i.built.Set(id) +} + +func (i isBuiltOrTouched) isBuilt(id uint32) bool { + return i.built.Has(id) +} + +func (i isBuiltOrTouched) setTouched(id uint32) { + i.touched.Set(id) +} + +func (i isBuiltOrTouched) isTouched(id uint32) bool { + return i.touched.Has(id) +} + +type isBuiltOrTouchedProvider interface { + isBuilt(uint32) bool + isTouched(uint32) bool +} + +type key interface { + comparable + fmt.Stringer +} + +type optionsCompiler[C any] interface { + isStaleCompiled(C) bool + compileOptions(map[string]any, defaultOptionValues) (C, error) +} + +type optionsGetSetter[K, C any] interface { + isBuiltOrTouchedProvider + identity.IdentityProvider + // resource.StaleInfo + + Compiled() C + Key() K + Reset() + + Get(uint32) OptionsSetter + isStale() bool + currPrev() (map[string]any, map[string]any) +} + +type optionsGetSetters[K key, C any] []optionsGetSetter[K, C] + +type optionsMap[K key, C any] map[K]optionsGetSetter[K, C] + +type opts[K any, C optionsCompiler[C]] struct { + key K + h *optsHolder[C] + once lazy.OnceMore +} + +type optsHolder[C optionsCompiler[C]] struct { + optionsID string + + defaults defaultOptionValues + + // Keep track of one generation so we can detect changes. + // Note that most of this tracking is performed on the options/map level. + compiled C + compiledPrev C + compileErr error + + resetCounter uint32 + optsSetCounter uint32 + optsCurr map[string]any + optsPrev map[string]any + + isBuiltOrTouched +} + +type paramsOptions struct { + Params json.RawMessage +} + +func (s paramsOptions) isStaleCompiled(prev paramsOptions) bool { + return false +} + +func (s paramsOptions) compileOptions(m map[string]any, defaults defaultOptionValues) (paramsOptions, error) { + v := struct { + Params map[string]any + }{} + + if err := mapstructure.WeakDecode(m, &v); err != nil { + return paramsOptions{}, err + } + + paramsJSON, err := json.Marshal(v.Params) + if err != nil { + return paramsOptions{}, err + } + + return paramsOptions{ + Params: paramsJSON, + }, nil +} + +type scriptBatchTemplateContext struct { + opts optionsGetSetter[scriptID, scriptOptions] + Import string + Instances []scriptInstanceBatchTemplateContext +} + +func (s *scriptBatchTemplateContext) Export() string { + return s.opts.Compiled().Export +} + +func (c scriptBatchTemplateContext) MarshalJSON() (b []byte, err error) { + return json.Marshal(&struct { + ID string `json:"id"` + Instances []scriptInstanceBatchTemplateContext `json:"instances"` + }{ + ID: c.opts.Key().String(), + Instances: c.Instances, + }) +} + +func (b scriptBatchTemplateContext) RunnerJSON(i int) string { + script := fmt.Sprintf("Script%d", i) + + v := struct { + ID string `json:"id"` + + // Read-only live JavaScript binding. + Binding string `json:"binding"` + Instances []scriptInstanceBatchTemplateContext `json:"instances"` + }{ + b.opts.Key().String(), + script, + b.Instances, + } + + bb, err := json.Marshal(v) + if err != nil { + panic(err) + } + s := string(bb) + + // Remove the quotes to make it a valid JS object. + s = strings.ReplaceAll(s, fmt.Sprintf("%q", script), script) + + return s +} + +type scriptGroup struct { + mu sync.Mutex + id string + b *batcher + isBuiltOrTouched + dependencyManager identity.Manager + + scriptsOptions optionsMap[scriptID, scriptOptions] + instancesOptions optionsMap[instanceID, paramsOptions] + runnersOptions optionsMap[scriptID, scriptOptions] +} + +// For internal use only. +func (b *scriptGroup) GetDependencyManager() identity.Manager { + return b.dependencyManager +} + +// For internal use only. +func (b *scriptGroup) IdentifierBase() string { + return b.id +} + +func (s *scriptGroup) Instance(sid, id string) OptionsSetter { + if err := ValidateBatchID(sid, false); err != nil { + panic(err) + } + if err := ValidateBatchID(id, false); err != nil { + panic(err) + } + + s.mu.Lock() + defer s.mu.Unlock() + + iid := instanceID{scriptID: scriptID(sid), instanceID: id} + if v, found := s.instancesOptions[iid]; found { + return v.Get(s.b.buildCount) + } + + fullID := "instance_" + s.key() + "_" + iid.String() + + s.instancesOptions[iid] = newOpts[instanceID, paramsOptions]( + iid, + fullID, + defaultOptionValues{}, + ) + + return s.instancesOptions[iid].Get(s.b.buildCount) +} + +func (g *scriptGroup) Reset() { + for _, v := range g.scriptsOptions { + v.Reset() + } + for _, v := range g.instancesOptions { + v.Reset() + } + for _, v := range g.runnersOptions { + v.Reset() + } +} + +func (s *scriptGroup) Runner(id string) OptionsSetter { + if err := ValidateBatchID(id, false); err != nil { + panic(err) + } + + s.mu.Lock() + defer s.mu.Unlock() + sid := scriptID(id) + if v, found := s.runnersOptions[sid]; found { + return v.Get(s.b.buildCount) + } + + runnerIdentity := "runner_" + s.key() + "_" + id + + // A typical signature for a runner would be: + // export default function Run(scripts) {} + // The user can override the default export in the templates. + + s.runnersOptions[sid] = newOpts[scriptID, scriptOptions]( + sid, + runnerIdentity, + defaultOptionValues{ + defaultExport: "default", + }, + ) + + return s.runnersOptions[sid].Get(s.b.buildCount) +} + +func (s *scriptGroup) Script(id string) OptionsSetter { + if err := ValidateBatchID(id, false); err != nil { + panic(err) + } + + s.mu.Lock() + defer s.mu.Unlock() + sid := scriptID(id) + if v, found := s.scriptsOptions[sid]; found { + return v.Get(s.b.buildCount) + } + + scriptIdentity := "script_" + s.key() + "_" + id + + s.scriptsOptions[sid] = newOpts[scriptID, scriptOptions]( + sid, + scriptIdentity, + defaultOptionValues{ + defaultExport: "*", + }, + ) + + return s.scriptsOptions[sid].Get(s.b.buildCount) +} + +func (s *scriptGroup) isStale() bool { + for _, v := range s.scriptsOptions { + if v.isStale() { + return true + } + } + + for _, v := range s.instancesOptions { + if v.isStale() { + return true + } + } + + for _, v := range s.runnersOptions { + if v.isStale() { + return true + } + } + + return false +} + +func (v *scriptGroup) forEachIdentity( + f func(id identity.Identity) bool, +) bool { + if f(v) { + return true + } + for _, vv := range v.instancesOptions { + if f(vv.GetIdentity()) { + return true + } + } + + for _, vv := range v.scriptsOptions { + if f(vv.GetIdentity()) { + return true + } + } + + for _, vv := range v.runnersOptions { + if f(vv.GetIdentity()) { + return true + } + } + + return false +} + +func (s *scriptGroup) key() string { + return s.b.id + "_" + s.id +} + +func (g *scriptGroup) removeNotSet() bool { + currentBuildID := g.b.buildCount + if !g.isBuilt(currentBuildID) { + // This group was never accessed in this build. + return false + } + var removed bool + + if g.instancesOptions.isBuilt(currentBuildID) { + // A new instance has been set in this group for this build. + // Remove any instance that has not been set in this build. + for k, v := range g.instancesOptions { + if v.isBuilt(currentBuildID) { + continue + } + delete(g.instancesOptions, k) + removed = true + } + } + + if g.runnersOptions.isBuilt(currentBuildID) { + // A new runner has been set in this group for this build. + // Remove any runner that has not been set in this build. + for k, v := range g.runnersOptions { + if v.isBuilt(currentBuildID) { + continue + } + delete(g.runnersOptions, k) + removed = true + } + } + + if g.scriptsOptions.isBuilt(currentBuildID) { + // A new script has been set in this group for this build. + // Remove any script that has not been set in this build. + for k, v := range g.scriptsOptions { + if v.isBuilt(currentBuildID) { + continue + } + delete(g.scriptsOptions, k) + + // Also remove any instance with this ID. + for kk := range g.instancesOptions { + if kk.scriptID == k { + delete(g.instancesOptions, kk) + } + } + removed = true + } + } + + return removed +} + +func (g *scriptGroup) removeTouchedButNotSet() bool { + currentBuildID := g.b.buildCount + var removed bool + for k, v := range g.instancesOptions { + if v.isBuilt(currentBuildID) { + continue + } + if v.isTouched(currentBuildID) { + delete(g.instancesOptions, k) + removed = true + } + } + for k, v := range g.runnersOptions { + if v.isBuilt(currentBuildID) { + continue + } + if v.isTouched(currentBuildID) { + delete(g.runnersOptions, k) + removed = true + } + } + for k, v := range g.scriptsOptions { + if v.isBuilt(currentBuildID) { + continue + } + if v.isTouched(currentBuildID) { + delete(g.scriptsOptions, k) + removed = true + + // Also remove any instance with this ID. + for kk := range g.instancesOptions { + if kk.scriptID == k { + delete(g.instancesOptions, kk) + } + } + } + + } + return removed +} + +type scriptGroups map[string]*scriptGroup + +func (s scriptGroups) Sorted() []*scriptGroup { + var a []*scriptGroup + for _, v := range s { + a = append(a, v) + } + sort.Slice(a, func(i, j int) bool { + return a[i].id < a[j].id + }) + return a +} + +type scriptID string + +func (s scriptID) String() string { + return string(s) +} + +type scriptInstanceBatchTemplateContext struct { + opts optionsGetSetter[instanceID, paramsOptions] +} + +func (c scriptInstanceBatchTemplateContext) ID() string { + return c.opts.Key().instanceID +} + +func (c scriptInstanceBatchTemplateContext) MarshalJSON() (b []byte, err error) { + return json.Marshal(&struct { + ID string `json:"id"` + Params json.RawMessage `json:"params"` + }{ + ID: c.opts.Key().instanceID, + Params: c.opts.Compiled().Params, + }) +} + +type scriptOptions struct { + // The script to build. + Resource resource.Resource + + // The import context to use. + // Note that we will always fall back to the resource's own import context. + ImportContext resource.ResourceGetter + + // The export name to use for this script's group's runners (if any). + // If not set, the default export will be used. + Export string + + // Params marshaled to JSON. + Params json.RawMessage +} + +func (o *scriptOptions) Dir() string { + return path.Dir(resources.InternalResourceTargetPath(o.Resource)) +} + +func (s scriptOptions) IsZero() bool { + return s.Resource == nil +} + +func (s scriptOptions) isStaleCompiled(prev scriptOptions) bool { + if prev.IsZero() { + return false + } + + // All but the ImportContext are checked at the options/map level. + i1nil, i2nil := prev.ImportContext == nil, s.ImportContext == nil + if i1nil && i2nil { + return false + } + if i1nil || i2nil { + return true + } + // On its own this check would have too many false positives, but combined with the other checks it should be fine. + // We cannot do equality checking here. + if !prev.ImportContext.(resource.IsProbablySameResourceGetter).IsProbablySameResourceGetter(s.ImportContext) { + return true + } + + return false +} + +func (s scriptOptions) compileOptions(m map[string]any, defaults defaultOptionValues) (scriptOptions, error) { + v := struct { + Resource resource.Resource + ImportContext any + Export string + Params map[string]any + }{} + + if err := mapstructure.WeakDecode(m, &v); err != nil { + panic(err) + } + + var paramsJSON []byte + if v.Params != nil { + var err error + paramsJSON, err = json.Marshal(v.Params) + if err != nil { + panic(err) + } + } + + if v.Export == "" { + v.Export = defaults.defaultExport + } + + compiled := scriptOptions{ + Resource: v.Resource, + Export: v.Export, + ImportContext: resource.NewCachedResourceGetter(v.ImportContext), + Params: paramsJSON, + } + + if compiled.Resource == nil { + return scriptOptions{}, fmt.Errorf("resource not set") + } + + return compiled, nil +} + +type scriptRunnerTemplateContext struct { + opts optionsGetSetter[scriptID, scriptOptions] + Import string +} + +func (s *scriptRunnerTemplateContext) Export() string { + return s.opts.Compiled().Export +} + +func (c scriptRunnerTemplateContext) MarshalJSON() (b []byte, err error) { + return json.Marshal(&struct { + ID string `json:"id"` + }{ + ID: c.opts.Key().String(), + }) +} + +func (o optionsMap[K, C]) isBuilt(id uint32) bool { + for _, v := range o { + if v.isBuilt(id) { + return true + } + } + + return false +} + +func (o *opts[K, C]) isBuilt(id uint32) bool { + return o.h.isBuilt(id) +} + +func (o *opts[K, C]) isStale() bool { + if o.h.isStaleOpts() { + return true + } + if o.h.compiled.isStaleCompiled(o.h.compiledPrev) { + return true + } + return false +} + +func (o *optsHolder[C]) isStaleOpts() bool { + if o.optsSetCounter == 1 && o.resetCounter > 0 { + return false + } + isStale := func() bool { + if len(o.optsCurr) != len(o.optsPrev) { + return true + } + for k, v := range o.optsPrev { + vv, found := o.optsCurr[k] + if !found { + return true + } + if strings.EqualFold(k, propsKeyImportContext) { + // This is checked later. + } else if si, ok := vv.(resource.StaleInfo); ok { + if si.StaleVersion() > 0 { + return true + } + } else { + if !reflect.DeepEqual(v, vv) { + return true + } + } + } + return false + }() + + return isStale +} + +func (o *opts[K, C]) isTouched(id uint32) bool { + return o.h.isTouched(id) +} + +func (o *optsHolder[C]) checkCompileErr() { + if o.compileErr != nil { + panic(o.compileErr) + } +} + +func (o *opts[K, C]) currPrev() (map[string]any, map[string]any) { + return o.h.optsCurr, o.h.optsPrev +} + +func init() { + // We don't want any dependencies/change tracking on the top level Package, + // we want finer grained control via Package.Group. + var p any = &Package{} + if _, ok := p.(identity.Identity); ok { + panic("esbuid.Package should not implement identity.Identity") + } + if _, ok := p.(identity.DependencyManagerProvider); ok { + panic("esbuid.Package should not implement identity.DependencyManagerProvider") + } +} diff --git a/internal/js/esbuild/batch_integration_test.go b/internal/js/esbuild/batch_integration_test.go new file mode 100644 index 000000000..07f99ee4e --- /dev/null +++ b/internal/js/esbuild/batch_integration_test.go @@ -0,0 +1,686 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package js provides functions for building JavaScript resources +package esbuild_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + qt "github.com/frankban/quicktest" + + "github.com/bep/logg" + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/hugolib" + "github.com/gohugoio/hugo/internal/js/esbuild" +) + +// Used to test misc. error situations etc. +const jsBatchFilesTemplate = ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "section"] +disableLiveReload = true +-- assets/js/styles.css -- +body { + background-color: red; +} +-- assets/js/main.js -- +import './styles.css'; +import * as params from '@params'; +import * as foo from 'mylib'; +console.log("Hello, Main!"); +console.log("params.p1", params.p1); +export default function Main() {}; +-- assets/js/runner.js -- +console.log("Hello, Runner!"); +-- node_modules/mylib/index.js -- +console.log("Hello, My Lib!"); +-- layouts/shortcodes/hdx.html -- +{{ $path := .Get "r" }} +{{ $r := or (.Page.Resources.Get $path) (resources.Get $path) }} +{{ $batch := (js.Batch "mybatch") }} +{{ $scriptID := $path | anchorize }} +{{ $instanceID := .Ordinal | string }} +{{ $group := .Page.RelPermalink | anchorize }} +{{ $params := .Params | default dict }} +{{ $export := .Get "export" | default "default" }} +{{ with $batch.Group $group }} + {{ with .Runner "create-elements" }} + {{ .SetOptions (dict "resource" (resources.Get "js/runner.js")) }} + {{ end }} + {{ with .Script $scriptID }} + {{ .SetOptions (dict + "resource" $r + "export" $export + "importContext" (slice $.Page) + ) + }} + {{ end }} + {{ with .Instance $scriptID $instanceID }} + {{ .SetOptions (dict "params" $params) }} + {{ end }} +{{ end }} +hdx-instance: {{ $scriptID }}: {{ $instanceID }}| +-- layouts/_default/baseof.html -- +Base. +{{ $batch := (js.Batch "mybatch") }} + {{ with $batch.Config }} + {{ .SetOptions (dict + "params" (dict "id" "config") + "sourceMap" "" + ) + }} +{{ end }} +{{ with (templates.Defer (dict "key" "global")) }} +Defer: +{{ $batch := (js.Batch "mybatch") }} +{{ range $k, $v := $batch.Build.Groups }} + {{ range $kk, $vv := . -}} + {{ $k }}: {{ .RelPermalink }} + {{ end }} +{{ end -}} +{{ end }} +{{ block "main" . }}Main{{ end }} +End. +-- layouts/_default/single.html -- +{{ define "main" }} +==> Single Template Content: {{ .Content }}$ +{{ $batch := (js.Batch "mybatch") }} +{{ with $batch.Group "mygroup" }} + {{ with .Runner "run" }} + {{ .SetOptions (dict "resource" (resources.Get "js/runner.js")) }} + {{ end }} + {{ with .Script "main" }} + {{ .SetOptions (dict "resource" (resources.Get "js/main.js") "params" (dict "p1" "param-p1-main" )) }} + {{ end }} + {{ with .Instance "main" "i1" }} + {{ .SetOptions (dict "params" (dict "title" "Instance 1")) }} + {{ end }} +{{ end }} +{{ end }} +-- layouts/index.html -- +{{ define "main" }} +Home. +{{ end }} +-- content/p1/index.md -- +--- +title: "P1" +--- + +Some content. + +{{< hdx r="p1script.js" myparam="p1-param-1" >}} +{{< hdx r="p1script.js" myparam="p1-param-2" >}} + +-- content/p1/p1script.js -- +console.log("P1 Script"); + + +` + +// Just to verify that the above file setup works. +func TestBatchTemplateOKBuild(t *testing.T) { + b := hugolib.Test(t, jsBatchFilesTemplate, hugolib.TestOptWithOSFs()) + b.AssertPublishDir("mybatch/mygroup.js", "mybatch/mygroup.css") +} + +func TestBatchRemoveAllInGroup(t *testing.T) { + files := jsBatchFilesTemplate + b := hugolib.TestRunning(t, files, hugolib.TestOptWithOSFs()) + + b.AssertFileContent("public/p1/index.html", "p1: /mybatch/p1.js") + + b.EditFiles("content/p1/index.md", ` +--- +title: "P1" +--- +Empty. +`) + b.Build() + + b.AssertFileContent("public/p1/index.html", "! p1: /mybatch/p1.js") + + // Add one script back. + b.EditFiles("content/p1/index.md", ` +--- +title: "P1" +--- + +{{< hdx r="p1script.js" myparam="p1-param-1-new" >}} +`) + b.Build() + + b.AssertFileContent("public/mybatch/p1.js", + "p1-param-1-new", + "p1script.js") +} + +func TestBatchEditInstance(t *testing.T) { + files := jsBatchFilesTemplate + b := hugolib.TestRunning(t, files, hugolib.TestOptWithOSFs()) + b.AssertFileContent("public/mybatch/mygroup.js", "Instance 1") + b.EditFileReplaceAll("layouts/_default/single.html", "Instance 1", "Instance 1 Edit").Build() + b.AssertFileContent("public/mybatch/mygroup.js", "Instance 1 Edit") +} + +func TestBatchEditScriptParam(t *testing.T) { + files := jsBatchFilesTemplate + b := hugolib.TestRunning(t, files, hugolib.TestOptWithOSFs()) + b.AssertFileContent("public/mybatch/mygroup.js", "param-p1-main") + b.EditFileReplaceAll("layouts/_default/single.html", "param-p1-main", "param-p1-main-edited").Build() + b.AssertFileContent("public/mybatch/mygroup.js", "param-p1-main-edited") +} + +func TestBatchErrorScriptResourceNotSet(t *testing.T) { + files := strings.Replace(jsBatchFilesTemplate, `(resources.Get "js/main.js")`, `(resources.Get "js/doesnotexist.js")`, 1) + b, err := hugolib.TestE(t, files, hugolib.TestOptWithOSFs()) + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, `error calling SetOptions: resource not set`) +} + +func TestBatchSlashInBatchID(t *testing.T) { + files := strings.ReplaceAll(jsBatchFilesTemplate, `"mybatch"`, `"my/batch"`) + b, err := hugolib.TestE(t, files, hugolib.TestOptWithOSFs()) + b.Assert(err, qt.IsNil) + b.AssertPublishDir("my/batch/mygroup.js") +} + +func TestBatchSourceMaps(t *testing.T) { + filesTemplate := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "section"] +disableLiveReload = true +-- assets/js/styles.css -- +body { + background-color: red; +} +-- assets/js/main.js -- +import * as foo from 'mylib'; +console.log("Hello, Main!"); +-- assets/js/runner.js -- +console.log("Hello, Runner!"); +-- node_modules/mylib/index.js -- +console.log("Hello, My Lib!"); +-- layouts/shortcodes/hdx.html -- +{{ $path := .Get "r" }} +{{ $r := or (.Page.Resources.Get $path) (resources.Get $path) }} +{{ $batch := (js.Batch "mybatch") }} +{{ $scriptID := $path | anchorize }} +{{ $instanceID := .Ordinal | string }} +{{ $group := .Page.RelPermalink | anchorize }} +{{ $params := .Params | default dict }} +{{ $export := .Get "export" | default "default" }} +{{ with $batch.Group $group }} + {{ with .Runner "create-elements" }} + {{ .SetOptions (dict "resource" (resources.Get "js/runner.js")) }} + {{ end }} + {{ with .Script $scriptID }} + {{ .SetOptions (dict + "resource" $r + "export" $export + "importContext" (slice $.Page) + ) + }} + {{ end }} + {{ with .Instance $scriptID $instanceID }} + {{ .SetOptions (dict "params" $params) }} + {{ end }} +{{ end }} +hdx-instance: {{ $scriptID }}: {{ $instanceID }}| +-- layouts/_default/baseof.html -- +Base. +{{ $batch := (js.Batch "mybatch") }} + {{ with $batch.Config }} + {{ .SetOptions (dict + "params" (dict "id" "config") + "sourceMap" "" + ) + }} +{{ end }} +{{ with (templates.Defer (dict "key" "global")) }} +Defer: +{{ $batch := (js.Batch "mybatch") }} +{{ range $k, $v := $batch.Build.Groups }} + {{ range $kk, $vv := . -}} + {{ $k }}: {{ .RelPermalink }} + {{ end }} +{{ end -}} +{{ end }} +{{ block "main" . }}Main{{ end }} +End. +-- layouts/_default/single.html -- +{{ define "main" }} +==> Single Template Content: {{ .Content }}$ +{{ $batch := (js.Batch "mybatch") }} +{{ with $batch.Group "mygroup" }} + {{ with .Runner "run" }} + {{ .SetOptions (dict "resource" (resources.Get "js/runner.js")) }} + {{ end }} + {{ with .Script "main" }} + {{ .SetOptions (dict "resource" (resources.Get "js/main.js") "params" (dict "p1" "param-p1-main" )) }} + {{ end }} + {{ with .Instance "main" "i1" }} + {{ .SetOptions (dict "params" (dict "title" "Instance 1")) }} + {{ end }} +{{ end }} +{{ end }} +-- layouts/index.html -- +{{ define "main" }} +Home. +{{ end }} +-- content/p1/index.md -- +--- +title: "P1" +--- + +Some content. + +{{< hdx r="p1script.js" myparam="p1-param-1" >}} +{{< hdx r="p1script.js" myparam="p1-param-2" >}} + +-- content/p1/p1script.js -- +import * as foo from 'mylib'; +console.lg("Foo", foo); +console.log("P1 Script"); +export default function P1Script() {}; + + +` + files := strings.Replace(filesTemplate, `"sourceMap" ""`, `"sourceMap" "linked"`, 1) + b := hugolib.TestRunning(t, files, hugolib.TestOptWithOSFs()) + b.AssertFileContent("public/mybatch/mygroup.js.map", "main.js", "! ns-hugo") + b.AssertFileContent("public/mybatch/mygroup.js", "sourceMappingURL=mygroup.js.map") + b.AssertFileContent("public/mybatch/p1.js", "sourceMappingURL=p1.js.map") + b.AssertFileContent("public/mybatch/mygroup_run_runner.js", "sourceMappingURL=mygroup_run_runner.js.map") + b.AssertFileContent("public/mybatch/chunk-UQKPPNA6.js", "sourceMappingURL=chunk-UQKPPNA6.js.map") + + checkMap := func(p string, expectLen int) { + s := b.FileContent(p) + sources := esbuild.SourcesFromSourceMap(s) + b.Assert(sources, qt.HasLen, expectLen) + + // Check that all source files exist. + for _, src := range sources { + filename, ok := paths.UrlStringToFilename(src) + b.Assert(ok, qt.IsTrue) + _, err := os.Stat(filename) + b.Assert(err, qt.IsNil) + } + } + + checkMap("public/mybatch/mygroup.js.map", 1) + checkMap("public/mybatch/p1.js.map", 1) + checkMap("public/mybatch/mygroup_run_runner.js.map", 0) + checkMap("public/mybatch/chunk-UQKPPNA6.js.map", 1) +} + +func TestBatchErrorRunnerResourceNotSet(t *testing.T) { + files := strings.Replace(jsBatchFilesTemplate, `(resources.Get "js/runner.js")`, `(resources.Get "js/doesnotexist.js")`, 1) + b, err := hugolib.TestE(t, files, hugolib.TestOptWithOSFs()) + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, `resource not set`) +} + +func TestBatchErrorScriptResourceInAssetsSyntaxError(t *testing.T) { + // Introduce JS syntax error in assets/js/main.js + files := strings.Replace(jsBatchFilesTemplate, `console.log("Hello, Main!");`, `console.log("Hello, Main!"`, 1) + b, err := hugolib.TestE(t, files, hugolib.TestOptWithOSFs()) + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, filepath.FromSlash(`assets/js/main.js:5:0": Expected ")" but found "console"`)) +} + +func TestBatchErrorScriptResourceInBundleSyntaxError(t *testing.T) { + // Introduce JS syntax error in content/p1/p1script.js + files := strings.Replace(jsBatchFilesTemplate, `console.log("P1 Script");`, `console.log("P1 Script"`, 1) + b, err := hugolib.TestE(t, files, hugolib.TestOptWithOSFs()) + b.Assert(err, qt.IsNotNil) + b.Assert(err.Error(), qt.Contains, filepath.FromSlash(`/content/p1/p1script.js:3:0": Expected ")" but found end of file`)) +} + +func TestBatch(t *testing.T) { + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term"] +disableLiveReload = true +baseURL = "https://example.com" +-- package.json -- +{ + "devDependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1" + } +} +-- assets/js/shims/react.js -- +-- assets/js/shims/react-dom.js -- +module.exports = window.ReactDOM; +module.exports = window.React; +-- content/mybundle/index.md -- +--- +title: "My Bundle" +--- +-- content/mybundle/mybundlestyles.css -- +@import './foo.css'; +@import './bar.css'; +@import './otherbundlestyles.css'; + +.mybundlestyles { + background-color: blue; +} +-- content/mybundle/bundlereact.jsx -- +import * as React from "react"; +import './foo.css'; +import './mybundlestyles.css'; +window.React1 = React; + +let text = 'Click me, too!' + +export default function MyBundleButton() { + return ( + <button>${text}</button> + ) +} + +-- assets/js/reactrunner.js -- +import * as ReactDOM from 'react-dom/client'; +import * as React from 'react'; + +export default function Run(group) { + for (const module of group.scripts) { + for (const instance of module.instances) { + /* This is a convention in this project. */ + let elId = §§${module.id}-${instance.id}§§; + let el = document.getElementById(elId); + if (!el) { + console.warn(§§Element with id ${elId} not found§§); + continue; + } + const root = ReactDOM.createRoot(el); + const reactEl = React.createElement(module.mod, instance.params); + root.render(reactEl); + } + } +} +-- assets/other/otherbundlestyles.css -- +.otherbundlestyles { + background-color: red; +} +-- assets/other/foo.css -- +@import './bar.css'; + +.foo { + background-color: blue; +} +-- assets/other/bar.css -- +.bar { + background-color: red; +} +-- assets/js/button.css -- +button { + background-color: red; +} +-- assets/js/bar.css -- +.bar-assets { + background-color: red; +} +-- assets/js/helper.js -- +import './bar.css' + +export function helper() { + console.log('helper'); +} + +-- assets/js/react1styles_nested.css -- +.react1styles_nested { + background-color: red; +} +-- assets/js/react1styles.css -- +@import './react1styles_nested.css'; +.react1styles { + background-color: red; +} +-- assets/js/react1.jsx -- +import * as React from "react"; +import './button.css' +import './foo.css' +import './react1styles.css' + +window.React1 = React; + +let text = 'Click me' + +export default function MyButton() { + return ( + <button>${text}</button> + ) +} + +-- assets/js/react2.jsx -- +import * as React from "react"; +import { helper } from './helper.js' +import './foo.css' + +window.React2 = React; + +let text = 'Click me, too!' + +export function MyOtherButton() { + return ( + <button>${text}</button> + ) +} +-- assets/js/main1.js -- +import * as React from "react"; +import * as params from '@params'; + +console.log('main1.React', React) +console.log('main1.params.id', params.id) + +-- assets/js/main2.js -- +import * as React from "react"; +import * as params from '@params'; + +console.log('main2.React', React) +console.log('main2.params.id', params.id) + +export default function Main2() {}; + +-- assets/js/main3.js -- +import * as React from "react"; +import * as params from '@params'; +import * as config from '@params/config'; + +console.log('main3.params.id', params.id) +console.log('config.params.id', config.id) + +export default function Main3() {}; + +-- layouts/_default/single.html -- +Single. + +{{ $r := .Resources.GetMatch "*.jsx" }} +{{ $batch := (js.Batch "mybundle") }} +{{ $otherCSS := (resources.Match "/other/*.css").Mount "/other" "." }} + {{ with $batch.Config }} + {{ $shims := dict "react" "js/shims/react.js" "react-dom/client" "js/shims/react-dom.js" }} + {{ .SetOptions (dict + "target" "es2018" + "params" (dict "id" "config") + "shims" $shims + ) + }} +{{ end }} +{{ with $batch.Group "reactbatch" }} + {{ with .Script "r3" }} + {{ .SetOptions (dict + "resource" $r + "importContext" (slice $ $otherCSS) + "params" (dict "id" "r3") + ) + }} + {{ end }} + {{ with .Instance "r3" "r2i1" }} + {{ .SetOptions (dict "title" "r2 instance 1")}} + {{ end }} +{{ end }} +-- layouts/index.html -- +Home. +{{ with (templates.Defer (dict "key" "global")) }} +{{ $batch := (js.Batch "mybundle") }} +{{ range $k, $v := $batch.Build.Groups }} + {{ range $kk, $vv := . }} + {{ $k }}: {{ $kk }}: {{ .RelPermalink }} + {{ end }} + {{ end }} +{{ end }} +{{ $myContentBundle := site.GetPage "mybundle" }} +{{ $batch := (js.Batch "mybundle") }} +{{ $otherCSS := (resources.Match "/other/*.css").Mount "/other" "." }} +{{ with $batch.Group "mains" }} + {{ with .Script "main1" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/main1.js") + "params" (dict "id" "main1") + ) + }} + {{ end }} + {{ with .Script "main2" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/main2.js") + "params" (dict "id" "main2") + ) + }} + {{ end }} + {{ with .Script "main3" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/main3.js") + ) + }} + {{ end }} +{{ with .Instance "main1" "m1i1" }}{{ .SetOptions (dict "params" (dict "title" "Main1 Instance 1"))}}{{ end }} +{{ with .Instance "main1" "m1i2" }}{{ .SetOptions (dict "params" (dict "title" "Main1 Instance 2"))}}{{ end }} +{{ end }} +{{ with $batch.Group "reactbatch" }} + {{ with .Runner "reactrunner" }} + {{ .SetOptions ( dict "resource" (resources.Get "js/reactrunner.js") )}} + {{ end }} + {{ with .Script "r1" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/react1.jsx") + "importContext" (slice $myContentBundle $otherCSS) + "params" (dict "id" "r1") + ) + }} + {{ end }} + {{ with .Instance "r1" "i1" }}{{ .SetOptions (dict "params" (dict "title" "Instance 1"))}}{{ end }} + {{ with .Instance "r1" "i2" }}{{ .SetOptions (dict "params" (dict "title" "Instance 2"))}}{{ end }} + {{ with .Script "r2" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/react2.jsx") + "export" "MyOtherButton" + "importContext" $otherCSS + "params" (dict "id" "r2") + ) + }} + {{ end }} + {{ with .Instance "r2" "i1" }}{{ .SetOptions (dict "params" (dict "title" "Instance 2-1"))}}{{ end }} +{{ end }} + +` + + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{ + T: t, + NeedsOsFS: true, + NeedsNpmInstall: true, + TxtarString: files, + Running: true, + LogLevel: logg.LevelWarn, + // PrintAndKeepTempDir: true, + }).Build() + + b.AssertFileContent("public/index.html", + "mains: 0: /mybundle/mains.js", + "reactbatch: 2: /mybundle/reactbatch.css", + ) + + b.AssertFileContent("public/mybundle/reactbatch.css", + ".bar {", + ) + + // Verify params resolution. + b.AssertFileContent("public/mybundle/mains.js", + ` +var id = "main1"; +console.log("main1.params.id", id); +var id2 = "main2"; +console.log("main2.params.id", id2); + + +# Params from top level config. +var id3 = "config"; +console.log("main3.params.id", void 0); +console.log("config.params.id", id3); +`) + + b.EditFileReplaceAll("content/mybundle/mybundlestyles.css", ".mybundlestyles", ".mybundlestyles-edit").Build() + b.AssertFileContent("public/mybundle/reactbatch.css", ".mybundlestyles-edit {") + + b.EditFileReplaceAll("assets/other/bar.css", ".bar {", ".bar-edit {").Build() + b.AssertFileContent("public/mybundle/reactbatch.css", ".bar-edit {") + + b.EditFileReplaceAll("assets/other/bar.css", ".bar-edit {", ".bar-edit2 {").Build() + b.AssertFileContent("public/mybundle/reactbatch.css", ".bar-edit2 {") +} + +func TestEditBaseofManyTimes(t *testing.T) { + files := ` +-- hugo.toml -- +baseURL = "https://example.com" +disableLiveReload = true +disableKinds = ["taxonomy", "term"] +-- layouts/_default/baseof.html -- +Baseof. +{{ block "main" . }}{{ end }} +{{ with (templates.Defer (dict "key" "global")) }} +Now. {{ now }} +{{ end }} +-- layouts/_default/single.html -- +{{ define "main" }} +Single. +{{ end }} +-- +-- layouts/_default/list.html -- +{{ define "main" }} +List. +{{ end }} +-- content/mybundle/index.md -- +--- +title: "My Bundle" +--- +-- content/_index.md -- +--- +title: "Home" +--- +` + + b := hugolib.TestRunning(t, files) + b.AssertFileContent("public/index.html", "Baseof.") + + for i := 0; i < 100; i++ { + b.EditFileReplaceAll("layouts/_default/baseof.html", "Now", "Now.").Build() + b.AssertFileContent("public/index.html", "Now..") + } +} diff --git a/internal/js/esbuild/build.go b/internal/js/esbuild/build.go new file mode 100644 index 000000000..33b91eafc --- /dev/null +++ b/internal/js/esbuild/build.go @@ -0,0 +1,236 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package esbuild provides functions for building JavaScript resources. +package esbuild + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/evanw/esbuild/pkg/api" + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/text" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/resources" +) + +// NewBuildClient creates a new BuildClient. +func NewBuildClient(fs *filesystems.SourceFilesystem, rs *resources.Spec) *BuildClient { + return &BuildClient{ + rs: rs, + sfs: fs, + } +} + +// BuildClient is a client for building JavaScript resources using esbuild. +type BuildClient struct { + rs *resources.Spec + sfs *filesystems.SourceFilesystem +} + +// Build builds the given JavaScript resources using esbuild with the given options. +func (c *BuildClient) Build(opts Options) (api.BuildResult, error) { + dependencyManager := opts.DependencyManager + if dependencyManager == nil { + dependencyManager = identity.NopManager + } + + opts.OutDir = c.rs.AbsPublishDir + opts.ResolveDir = c.rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved + opts.AbsWorkingDir = opts.ResolveDir + opts.TsConfig = c.rs.ResolveJSConfigFile("tsconfig.json") + assetsResolver := newFSResolver(c.rs.Assets.Fs) + + if err := opts.validate(); err != nil { + return api.BuildResult{}, err + } + + if err := opts.compile(); err != nil { + return api.BuildResult{}, err + } + + var err error + opts.compiled.Plugins, err = createBuildPlugins(c.rs, assetsResolver, dependencyManager, opts) + if err != nil { + return api.BuildResult{}, err + } + + if opts.Inject != nil { + // Resolve the absolute filenames. + for i, ext := range opts.Inject { + impPath := filepath.FromSlash(ext) + if filepath.IsAbs(impPath) { + return api.BuildResult{}, fmt.Errorf("inject: absolute paths not supported, must be relative to /assets") + } + + m := assetsResolver.resolveComponent(impPath) + + if m == nil { + return api.BuildResult{}, fmt.Errorf("inject: file %q not found", ext) + } + + opts.Inject[i] = m.Filename + + } + + opts.compiled.Inject = opts.Inject + + } + + result := api.Build(opts.compiled) + + if len(result.Errors) > 0 { + createErr := func(msg api.Message) error { + if msg.Location == nil { + return errors.New(msg.Text) + } + var ( + contentr hugio.ReadSeekCloser + errorMessage string + loc = msg.Location + errorPath = loc.File + err error + ) + + var resolvedError *ErrorMessageResolved + + if opts.ErrorMessageResolveFunc != nil { + resolvedError = opts.ErrorMessageResolveFunc(msg) + } + + if resolvedError == nil { + if errorPath == stdinImporter { + errorPath = opts.StdinSourcePath + } + + errorMessage = msg.Text + + var namespace string + for _, ns := range hugoNamespaces { + if strings.HasPrefix(errorPath, ns) { + namespace = ns + break + } + } + + if namespace != "" { + namespace += ":" + errorMessage = strings.ReplaceAll(errorMessage, namespace, "") + errorPath = strings.TrimPrefix(errorPath, namespace) + contentr, err = hugofs.Os.Open(errorPath) + } else { + var fi os.FileInfo + fi, err = c.sfs.Fs.Stat(errorPath) + if err == nil { + m := fi.(hugofs.FileMetaInfo).Meta() + errorPath = m.Filename + contentr, err = m.Open() + } + } + } else { + contentr = resolvedError.Content + errorPath = resolvedError.Path + errorMessage = resolvedError.Message + } + + if contentr != nil { + defer contentr.Close() + } + + if err == nil { + fe := herrors. + NewFileErrorFromName(errors.New(errorMessage), errorPath). + UpdatePosition(text.Position{Offset: -1, LineNumber: loc.Line, ColumnNumber: loc.Column}). + UpdateContent(contentr, nil) + + return fe + } + + return fmt.Errorf("%s", errorMessage) + } + + var errors []error + + for _, msg := range result.Errors { + errors = append(errors, createErr(msg)) + } + + // Return 1, log the rest. + for i, err := range errors { + if i > 0 { + c.rs.Logger.Errorf("js.Build failed: %s", err) + } + } + + return result, errors[0] + } + + inOutputPathToAbsFilename := opts.ResolveSourceMapSource + opts.ResolveSourceMapSource = func(s string) string { + if inOutputPathToAbsFilename != nil { + if filename := inOutputPathToAbsFilename(s); filename != "" { + return filename + } + } + + if m := assetsResolver.resolveComponent(s); m != nil { + return m.Filename + } + + return "" + } + + for i, o := range result.OutputFiles { + if err := fixOutputFile(&o, func(s string) string { + if s == "<stdin>" { + return opts.ResolveSourceMapSource(opts.StdinSourcePath) + } + var isNsHugo bool + if strings.HasPrefix(s, "ns-hugo") { + isNsHugo = true + idxColon := strings.Index(s, ":") + s = s[idxColon+1:] + } + + if !strings.HasPrefix(s, PrefixHugoVirtual) { + if !filepath.IsAbs(s) { + s = filepath.Join(opts.OutDir, s) + } + } + + if isNsHugo { + if ss := opts.ResolveSourceMapSource(s); ss != "" { + if strings.HasPrefix(ss, PrefixHugoMemory) { + // File not on disk, mark it for removal from the sources slice. + return "" + } + return ss + } + return "" + } + return s + }); err != nil { + return result, err + } + result.OutputFiles[i] = o + } + + return result, nil +} diff --git a/resources/resource_transformers/js/build_test.go b/internal/js/esbuild/helpers.go index 30a4490ed..b4cb565b8 100644 --- a/resources/resource_transformers/js/build_test.go +++ b/internal/js/esbuild/helpers.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,4 +11,5 @@ // See the License for the specific language governing permissions and // limitations under the License. -package js +// Package esbuild provides functions for building JavaScript resources. +package esbuild diff --git a/internal/js/esbuild/options.go b/internal/js/esbuild/options.go new file mode 100644 index 000000000..16fc0d4bb --- /dev/null +++ b/internal/js/esbuild/options.go @@ -0,0 +1,375 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package esbuild + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/identity" + + "github.com/evanw/esbuild/pkg/api" + + "github.com/gohugoio/hugo/media" + "github.com/mitchellh/mapstructure" +) + +var ( + nameTarget = map[string]api.Target{ + "": api.ESNext, + "esnext": api.ESNext, + "es5": api.ES5, + "es6": api.ES2015, + "es2015": api.ES2015, + "es2016": api.ES2016, + "es2017": api.ES2017, + "es2018": api.ES2018, + "es2019": api.ES2019, + "es2020": api.ES2020, + "es2021": api.ES2021, + "es2022": api.ES2022, + "es2023": api.ES2023, + } + + // source names: https://github.com/evanw/esbuild/blob/9eca46464ed5615cb36a3beb3f7a7b9a8ffbe7cf/internal/config/config.go#L208 + nameLoader = map[string]api.Loader{ + "none": api.LoaderNone, + "base64": api.LoaderBase64, + "binary": api.LoaderBinary, + "copy": api.LoaderFile, + "css": api.LoaderCSS, + "dataurl": api.LoaderDataURL, + "default": api.LoaderDefault, + "empty": api.LoaderEmpty, + "file": api.LoaderFile, + "global-css": api.LoaderGlobalCSS, + "js": api.LoaderJS, + "json": api.LoaderJSON, + "jsx": api.LoaderJSX, + "local-css": api.LoaderLocalCSS, + "text": api.LoaderText, + "ts": api.LoaderTS, + "tsx": api.LoaderTSX, + } +) + +// DecodeExternalOptions decodes the given map into ExternalOptions. +func DecodeExternalOptions(m map[string]any) (ExternalOptions, error) { + opts := ExternalOptions{ + SourcesContent: true, + } + + if err := mapstructure.WeakDecode(m, &opts); err != nil { + return opts, err + } + + if opts.TargetPath != "" { + opts.TargetPath = paths.ToSlashTrimLeading(opts.TargetPath) + } + + opts.Target = strings.ToLower(opts.Target) + opts.Format = strings.ToLower(opts.Format) + + return opts, nil +} + +// ErrorMessageResolved holds a resolved error message. +type ErrorMessageResolved struct { + Path string + Message string + Content hugio.ReadSeekCloser +} + +// ExternalOptions holds user facing options for the js.Build template function. +type ExternalOptions struct { + // If not set, the source path will be used as the base target path. + // Note that the target path's extension may change if the target MIME type + // is different, e.g. when the source is TypeScript. + TargetPath string + + // Whether to minify to output. + Minify bool + + // One of "inline", "external", "linked" or "none". + SourceMap string + + SourcesContent bool + + // The language target. + // One of: es2015, es2016, es2017, es2018, es2019, es2020 or esnext. + // Default is esnext. + Target string + + // The output format. + // One of: iife, cjs, esm + // Default is to esm. + Format string + + // External dependencies, e.g. "react". + Externals []string + + // This option allows you to automatically replace a global variable with an import from another file. + // The filenames must be relative to /assets. + // See https://esbuild.github.io/api/#inject + Inject []string + + // User defined symbols. + Defines map[string]any + + // Maps a component import to another. + Shims map[string]string + + // Configuring a loader for a given file type lets you load that file type with an + // import statement or a require call. For example, configuring the .png file extension + // to use the data URL loader means importing a .png file gives you a data URL + // containing the contents of that image + // + // See https://esbuild.github.io/api/#loader + Loaders map[string]string + + // User defined params. Will be marshaled to JSON and available as "@params", e.g. + // import * as params from '@params'; + Params any + + // What to use instead of React.createElement. + JSXFactory string + + // What to use instead of React.Fragment. + JSXFragment string + + // What to do about JSX syntax. + // See https://esbuild.github.io/api/#jsx + JSX string + + // Which library to use to automatically import JSX helper functions from. Only works if JSX is set to automatic. + // See https://esbuild.github.io/api/#jsx-import-source + JSXImportSource string + + // There is/was a bug in WebKit with severe performance issue with the tracking + // of TDZ checks in JavaScriptCore. + // + // Enabling this flag removes the TDZ and `const` assignment checks and + // may improve performance of larger JS codebases until the WebKit fix + // is in widespread use. + // + // See https://bugs.webkit.org/show_bug.cgi?id=199866 + // Deprecated: This no longer have any effect and will be removed. + // TODO(bep) remove. See https://github.com/evanw/esbuild/commit/869e8117b499ca1dbfc5b3021938a53ffe934dba + AvoidTDZ bool +} + +// InternalOptions holds internal options for the js.Build template function. +type InternalOptions struct { + MediaType media.Type + OutDir string + Contents string + SourceDir string + ResolveDir string + AbsWorkingDir string + Metafile bool + + StdinSourcePath string + + DependencyManager identity.Manager + + Stdin bool // Set to true to pass in the entry point as a byte slice. + Splitting bool + TsConfig string + EntryPoints []string + ImportOnResolveFunc func(string, api.OnResolveArgs) string + ImportOnLoadFunc func(api.OnLoadArgs) string + ImportParamsOnLoadFunc func(args api.OnLoadArgs) json.RawMessage + ErrorMessageResolveFunc func(api.Message) *ErrorMessageResolved + ResolveSourceMapSource func(string) string // Used to resolve paths in error source maps. +} + +// Options holds the options passed to Build. +type Options struct { + ExternalOptions + InternalOptions + + compiled api.BuildOptions +} + +func (opts *Options) compile() (err error) { + target, found := nameTarget[opts.Target] + if !found { + err = fmt.Errorf("invalid target: %q", opts.Target) + return + } + + var loaders map[string]api.Loader + if opts.Loaders != nil { + loaders = make(map[string]api.Loader) + for k, v := range opts.Loaders { + loader, found := nameLoader[v] + if !found { + err = fmt.Errorf("invalid loader: %q", v) + return + } + loaders[k] = loader + } + } + + mediaType := opts.MediaType + if mediaType.IsZero() { + mediaType = media.Builtin.JavascriptType + } + + var loader api.Loader + switch mediaType.SubType { + case media.Builtin.JavascriptType.SubType: + loader = api.LoaderJS + case media.Builtin.TypeScriptType.SubType: + loader = api.LoaderTS + case media.Builtin.TSXType.SubType: + loader = api.LoaderTSX + case media.Builtin.JSXType.SubType: + loader = api.LoaderJSX + default: + err = fmt.Errorf("unsupported Media Type: %q", opts.MediaType) + return + } + + var format api.Format + // One of: iife, cjs, esm + switch opts.Format { + case "", "iife": + format = api.FormatIIFE + case "esm": + format = api.FormatESModule + case "cjs": + format = api.FormatCommonJS + default: + err = fmt.Errorf("unsupported script output format: %q", opts.Format) + return + } + + var jsx api.JSX + switch opts.JSX { + case "", "transform": + jsx = api.JSXTransform + case "preserve": + jsx = api.JSXPreserve + case "automatic": + jsx = api.JSXAutomatic + default: + err = fmt.Errorf("unsupported jsx type: %q", opts.JSX) + return + } + + var defines map[string]string + if opts.Defines != nil { + defines = maps.ToStringMapString(opts.Defines) + } + + // By default we only need to specify outDir and no outFile + outDir := opts.OutDir + outFile := "" + var sourceMap api.SourceMap + switch opts.SourceMap { + case "inline": + sourceMap = api.SourceMapInline + case "external": + sourceMap = api.SourceMapExternal + case "linked": + sourceMap = api.SourceMapLinked + case "", "none": + sourceMap = api.SourceMapNone + default: + err = fmt.Errorf("unsupported sourcemap type: %q", opts.SourceMap) + return + } + + sourcesContent := api.SourcesContentInclude + if !opts.SourcesContent { + sourcesContent = api.SourcesContentExclude + } + + opts.compiled = api.BuildOptions{ + Outfile: outFile, + Bundle: true, + Metafile: opts.Metafile, + AbsWorkingDir: opts.AbsWorkingDir, + + Target: target, + Format: format, + Sourcemap: sourceMap, + SourcesContent: sourcesContent, + + Loader: loaders, + + MinifyWhitespace: opts.Minify, + MinifyIdentifiers: opts.Minify, + MinifySyntax: opts.Minify, + + Outdir: outDir, + Splitting: opts.Splitting, + + Define: defines, + External: opts.Externals, + + JSXFactory: opts.JSXFactory, + JSXFragment: opts.JSXFragment, + + JSX: jsx, + JSXImportSource: opts.JSXImportSource, + + Tsconfig: opts.TsConfig, + + EntryPoints: opts.EntryPoints, + } + + if opts.Stdin { + // This makes ESBuild pass `stdin` as the Importer to the import. + opts.compiled.Stdin = &api.StdinOptions{ + Contents: opts.Contents, + ResolveDir: opts.ResolveDir, + Loader: loader, + } + } + return +} + +func (o Options) loaderFromFilename(filename string) api.Loader { + ext := filepath.Ext(filename) + if optsLoaders := o.compiled.Loader; optsLoaders != nil { + if l, found := optsLoaders[ext]; found { + return l + } + } + l, found := extensionToLoaderMap[ext] + if found { + return l + } + return api.LoaderJS +} + +func (opts *Options) validate() error { + if opts.ImportOnResolveFunc != nil && opts.ImportOnLoadFunc == nil { + return fmt.Errorf("ImportOnLoadFunc must be set if ImportOnResolveFunc is set") + } + if opts.ImportOnResolveFunc == nil && opts.ImportOnLoadFunc != nil { + return fmt.Errorf("ImportOnResolveFunc must be set if ImportOnLoadFunc is set") + } + if opts.AbsWorkingDir == "" { + return fmt.Errorf("AbsWorkingDir must be set") + } + return nil +} diff --git a/internal/js/esbuild/options_test.go b/internal/js/esbuild/options_test.go new file mode 100644 index 000000000..ca19717f7 --- /dev/null +++ b/internal/js/esbuild/options_test.go @@ -0,0 +1,219 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package esbuild + +import ( + "testing" + + "github.com/gohugoio/hugo/media" + + "github.com/evanw/esbuild/pkg/api" + + qt "github.com/frankban/quicktest" +) + +func TestToBuildOptions(t *testing.T) { + c := qt.New(t) + + opts := Options{ + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + Stdin: true, + }, + } + + c.Assert(opts.compile(), qt.IsNil) + c.Assert(opts.compiled, qt.DeepEquals, api.BuildOptions{ + Bundle: true, + Target: api.ESNext, + Format: api.FormatIIFE, + SourcesContent: 1, + Stdin: &api.StdinOptions{ + Loader: api.LoaderJS, + }, + }) + + opts = Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", + Format: "cjs", + Minify: true, + AvoidTDZ: true, + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + Stdin: true, + }, + } + + c.Assert(opts.compile(), qt.IsNil) + c.Assert(opts.compiled, qt.DeepEquals, api.BuildOptions{ + Bundle: true, + Target: api.ES2018, + Format: api.FormatCommonJS, + SourcesContent: 1, + MinifyIdentifiers: true, + MinifySyntax: true, + MinifyWhitespace: true, + Stdin: &api.StdinOptions{ + Loader: api.LoaderJS, + }, + }) + + opts = Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "inline", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + Stdin: true, + }, + } + + c.Assert(opts.compile(), qt.IsNil) + c.Assert(opts.compiled, qt.DeepEquals, api.BuildOptions{ + Bundle: true, + Target: api.ES2018, + Format: api.FormatCommonJS, + MinifyIdentifiers: true, + MinifySyntax: true, + MinifyWhitespace: true, + SourcesContent: 1, + Sourcemap: api.SourceMapInline, + Stdin: &api.StdinOptions{ + Loader: api.LoaderJS, + }, + }) + + opts = Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "inline", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + Stdin: true, + }, + } + + c.Assert(opts.compile(), qt.IsNil) + c.Assert(opts.compiled, qt.DeepEquals, api.BuildOptions{ + Bundle: true, + Target: api.ES2018, + Format: api.FormatCommonJS, + MinifyIdentifiers: true, + MinifySyntax: true, + MinifyWhitespace: true, + Sourcemap: api.SourceMapInline, + SourcesContent: 1, + Stdin: &api.StdinOptions{ + Loader: api.LoaderJS, + }, + }) + + opts = Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "external", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + Stdin: true, + }, + } + + c.Assert(opts.compile(), qt.IsNil) + c.Assert(opts.compiled, qt.DeepEquals, api.BuildOptions{ + Bundle: true, + Target: api.ES2018, + Format: api.FormatCommonJS, + MinifyIdentifiers: true, + MinifySyntax: true, + MinifyWhitespace: true, + Sourcemap: api.SourceMapExternal, + SourcesContent: 1, + Stdin: &api.StdinOptions{ + Loader: api.LoaderJS, + }, + }) + + opts = Options{ + ExternalOptions: ExternalOptions{ + JSX: "automatic", JSXImportSource: "preact", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + Stdin: true, + }, + } + + c.Assert(opts.compile(), qt.IsNil) + c.Assert(opts.compiled, qt.DeepEquals, api.BuildOptions{ + Bundle: true, + Target: api.ESNext, + Format: api.FormatIIFE, + SourcesContent: 1, + Stdin: &api.StdinOptions{ + Loader: api.LoaderJS, + }, + JSX: api.JSXAutomatic, + JSXImportSource: "preact", + }) +} + +func TestToBuildOptionsTarget(t *testing.T) { + c := qt.New(t) + + for _, test := range []struct { + target string + expect api.Target + }{ + {"es2015", api.ES2015}, + {"es2016", api.ES2016}, + {"es2017", api.ES2017}, + {"es2018", api.ES2018}, + {"es2019", api.ES2019}, + {"es2020", api.ES2020}, + {"es2021", api.ES2021}, + {"es2022", api.ES2022}, + {"es2023", api.ES2023}, + {"", api.ESNext}, + {"esnext", api.ESNext}, + } { + c.Run(test.target, func(c *qt.C) { + opts := Options{ + ExternalOptions: ExternalOptions{ + Target: test.target, + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + } + + c.Assert(opts.compile(), qt.IsNil) + c.Assert(opts.compiled.Target, qt.Equals, test.expect) + }) + } +} + +func TestDecodeExternalOptions(t *testing.T) { + c := qt.New(t) + m := map[string]any{} + opts, err := DecodeExternalOptions(m) + c.Assert(err, qt.IsNil) + c.Assert(opts, qt.DeepEquals, ExternalOptions{ + SourcesContent: true, + }) +} diff --git a/internal/js/esbuild/resolve.go b/internal/js/esbuild/resolve.go new file mode 100644 index 000000000..ac0010da9 --- /dev/null +++ b/internal/js/esbuild/resolve.go @@ -0,0 +1,315 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package esbuild + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/evanw/esbuild/pkg/api" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/resource" + "github.com/spf13/afero" +) + +const ( + NsHugoImport = "ns-hugo-imp" + NsHugoImportResolveFunc = "ns-hugo-imp-func" + nsHugoParams = "ns-hugo-params" + pathHugoConfigParams = "@params/config" + + stdinImporter = "<stdin>" +) + +var hugoNamespaces = []string{NsHugoImport, NsHugoImportResolveFunc, nsHugoParams} + +const ( + PrefixHugoVirtual = "__hu_v" + PrefixHugoMemory = "__hu_m" +) + +var extensionToLoaderMap = map[string]api.Loader{ + ".js": api.LoaderJS, + ".mjs": api.LoaderJS, + ".cjs": api.LoaderJS, + ".jsx": api.LoaderJSX, + ".ts": api.LoaderTS, + ".tsx": api.LoaderTSX, + ".css": api.LoaderCSS, + ".json": api.LoaderJSON, + ".txt": api.LoaderText, +} + +// This is a common sub-set of ESBuild's default extensions. +// We assume that imports of JSON, CSS etc. will be using their full +// name with extension. +var commonExtensions = []string{".js", ".ts", ".tsx", ".jsx"} + +// ResolveComponent resolves a component using the given resolver. +func ResolveComponent[T any](impPath string, resolve func(string) (v T, found, isDir bool)) (v T, found bool) { + findFirst := func(base string) (v T, found, isDir bool) { + for _, ext := range commonExtensions { + if strings.HasSuffix(impPath, ext) { + // Import of foo.js.js need the full name. + continue + } + if v, found, isDir = resolve(base + ext); found { + return + } + } + + // Not found. + return + } + + // We need to check if this is a regular file imported without an extension. + // There may be ambiguous situations where both foo.js and foo/index.js exists. + // This import order is in line with both how Node and ESBuild's native + // import resolver works. + + // It may be a regular file imported without an extension, e.g. + // foo or foo/index. + v, found, _ = findFirst(impPath) + if found { + return v, found + } + + base := filepath.Base(impPath) + if base == "index" { + // try index.esm.js etc. + v, found, _ = findFirst(impPath + ".esm") + if found { + return v, found + } + } + + // Check the path as is. + var isDir bool + v, found, isDir = resolve(impPath) + if found && isDir { + v, found, _ = findFirst(filepath.Join(impPath, "index")) + if !found { + v, found, _ = findFirst(filepath.Join(impPath, "index.esm")) + } + } + + if !found && strings.HasSuffix(base, ".js") { + v, found, _ = findFirst(strings.TrimSuffix(impPath, ".js")) + } + + return +} + +// ResolveResource resolves a resource using the given resourceGetter. +func ResolveResource(impPath string, resourceGetter resource.ResourceGetter) (r resource.Resource) { + resolve := func(name string) (v resource.Resource, found, isDir bool) { + r := resourceGetter.Get(name) + return r, r != nil, false + } + r, found := ResolveComponent(impPath, resolve) + if !found { + return nil + } + return r +} + +func newFSResolver(fs afero.Fs) *fsResolver { + return &fsResolver{fs: fs, resolved: maps.NewCache[string, *hugofs.FileMeta]()} +} + +type fsResolver struct { + fs afero.Fs + resolved *maps.Cache[string, *hugofs.FileMeta] +} + +func (r *fsResolver) resolveComponent(impPath string) *hugofs.FileMeta { + v, _ := r.resolved.GetOrCreate(impPath, func() (*hugofs.FileMeta, error) { + resolve := func(name string) (*hugofs.FileMeta, bool, bool) { + if fi, err := r.fs.Stat(name); err == nil { + return fi.(hugofs.FileMetaInfo).Meta(), true, fi.IsDir() + } + return nil, false, false + } + v, _ := ResolveComponent(impPath, resolve) + return v, nil + }) + return v +} + +func createBuildPlugins(rs *resources.Spec, assetsResolver *fsResolver, depsManager identity.Manager, opts Options) ([]api.Plugin, error) { + fs := rs.Assets + + resolveImport := func(args api.OnResolveArgs) (api.OnResolveResult, error) { + impPath := args.Path + shimmed := false + if opts.Shims != nil { + override, found := opts.Shims[impPath] + if found { + impPath = override + shimmed = true + } + } + + if opts.ImportOnResolveFunc != nil { + if s := opts.ImportOnResolveFunc(impPath, args); s != "" { + return api.OnResolveResult{Path: s, Namespace: NsHugoImportResolveFunc}, nil + } + } + + importer := args.Importer + + isStdin := importer == stdinImporter + var relDir string + if !isStdin { + if strings.HasPrefix(importer, PrefixHugoVirtual) { + relDir = filepath.Dir(strings.TrimPrefix(importer, PrefixHugoVirtual)) + } else { + rel, found := fs.MakePathRelative(importer, true) + + if !found { + if shimmed { + relDir = opts.SourceDir + } else { + // Not in any of the /assets folders. + // This is an import from a node_modules, let + // ESBuild resolve this. + return api.OnResolveResult{}, nil + } + } else { + relDir = filepath.Dir(rel) + } + } + } else { + relDir = opts.SourceDir + } + + // Imports not starting with a "." is assumed to live relative to /assets. + // Hugo makes no assumptions about the directory structure below /assets. + if relDir != "" && strings.HasPrefix(impPath, ".") { + impPath = filepath.Join(relDir, impPath) + } + + m := assetsResolver.resolveComponent(impPath) + + if m != nil { + depsManager.AddIdentity(m.PathInfo) + + // Store the source root so we can create a jsconfig.json + // to help IntelliSense when the build is done. + // This should be a small number of elements, and when + // in server mode, we may get stale entries on renames etc., + // but that shouldn't matter too much. + rs.JSConfigBuilder.AddSourceRoot(m.SourceRoot) + return api.OnResolveResult{Path: m.Filename, Namespace: NsHugoImport}, nil + } + + // Fall back to ESBuild's resolve. + return api.OnResolveResult{}, nil + } + + importResolver := api.Plugin{ + Name: "hugo-import-resolver", + Setup: func(build api.PluginBuild) { + build.OnResolve(api.OnResolveOptions{Filter: `.*`}, + func(args api.OnResolveArgs) (api.OnResolveResult, error) { + return resolveImport(args) + }) + build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: NsHugoImport}, + func(args api.OnLoadArgs) (api.OnLoadResult, error) { + b, err := os.ReadFile(args.Path) + if err != nil { + return api.OnLoadResult{}, fmt.Errorf("failed to read %q: %w", args.Path, err) + } + c := string(b) + + return api.OnLoadResult{ + // See https://github.com/evanw/esbuild/issues/502 + // This allows all modules to resolve dependencies + // in the main project's node_modules. + ResolveDir: opts.ResolveDir, + Contents: &c, + Loader: opts.loaderFromFilename(args.Path), + }, nil + }) + build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: NsHugoImportResolveFunc}, + func(args api.OnLoadArgs) (api.OnLoadResult, error) { + c := opts.ImportOnLoadFunc(args) + if c == "" { + return api.OnLoadResult{}, fmt.Errorf("ImportOnLoadFunc failed to resolve %q", args.Path) + } + + return api.OnLoadResult{ + ResolveDir: opts.ResolveDir, + Contents: &c, + Loader: opts.loaderFromFilename(args.Path), + }, nil + }) + }, + } + + params := opts.Params + if params == nil { + // This way @params will always resolve to something. + params = make(map[string]any) + } + + b, err := json.Marshal(params) + if err != nil { + return nil, fmt.Errorf("failed to marshal params: %w", err) + } + + paramsPlugin := api.Plugin{ + Name: "hugo-params-plugin", + Setup: func(build api.PluginBuild) { + build.OnResolve(api.OnResolveOptions{Filter: `^@params(/config)?$`}, + func(args api.OnResolveArgs) (api.OnResolveResult, error) { + resolvedPath := args.Importer + + if args.Path == pathHugoConfigParams { + resolvedPath = pathHugoConfigParams + } + + return api.OnResolveResult{ + Path: resolvedPath, + Namespace: nsHugoParams, + }, nil + }) + build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: nsHugoParams}, + func(args api.OnLoadArgs) (api.OnLoadResult, error) { + bb := b + if args.Path != pathHugoConfigParams && opts.ImportParamsOnLoadFunc != nil { + bb = opts.ImportParamsOnLoadFunc(args) + } + s := string(bb) + + if s == "" { + s = "{}" + } + + return api.OnLoadResult{ + Contents: &s, + Loader: api.LoaderJSON, + }, nil + }) + }, + } + + return []api.Plugin{importResolver, paramsPlugin}, nil +} diff --git a/internal/js/esbuild/resolve_test.go b/internal/js/esbuild/resolve_test.go new file mode 100644 index 000000000..86e3138f2 --- /dev/null +++ b/internal/js/esbuild/resolve_test.go @@ -0,0 +1,86 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package esbuild + +import ( + "path" + "path/filepath" + "testing" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/testconfig" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/hugolib/paths" + "github.com/spf13/afero" +) + +func TestResolveComponentInAssets(t *testing.T) { + c := qt.New(t) + + for _, test := range []struct { + name string + files []string + impPath string + expect string + }{ + {"Basic, extension", []string{"foo.js", "bar.js"}, "foo.js", "foo.js"}, + {"Basic, no extension", []string{"foo.js", "bar.js"}, "foo", "foo.js"}, + {"Basic, no extension, typescript", []string{"foo.ts", "bar.js"}, "foo", "foo.ts"}, + {"Not found", []string{"foo.js", "bar.js"}, "moo.js", ""}, + {"Not found, double js extension", []string{"foo.js.js", "bar.js"}, "foo.js", ""}, + {"Index file, folder only", []string{"foo/index.js", "bar.js"}, "foo", "foo/index.js"}, + {"Index file, folder and index", []string{"foo/index.js", "bar.js"}, "foo/index", "foo/index.js"}, + {"Index file, folder and index and suffix", []string{"foo/index.js", "bar.js"}, "foo/index.js", "foo/index.js"}, + {"Index ESM file, folder only", []string{"foo/index.esm.js", "bar.js"}, "foo", "foo/index.esm.js"}, + {"Index ESM file, folder and index", []string{"foo/index.esm.js", "bar.js"}, "foo/index", "foo/index.esm.js"}, + {"Index ESM file, folder and index and suffix", []string{"foo/index.esm.js", "bar.js"}, "foo/index.esm.js", "foo/index.esm.js"}, + // We added these index.esm.js cases in v0.101.0. The case below is unlikely to happen in the wild, but add a test + // to document Hugo's behavior. We pick the file with the name index.js; anything else would be breaking. + {"Index and Index ESM file, folder only", []string{"foo/index.esm.js", "foo/index.js", "bar.js"}, "foo", "foo/index.js"}, + + // Issue #8949 + {"Check file before directory", []string{"foo.js", "foo/index.js"}, "foo", "foo.js"}, + } { + c.Run(test.name, func(c *qt.C) { + baseDir := "assets" + mfs := afero.NewMemMapFs() + + for _, filename := range test.files { + c.Assert(afero.WriteFile(mfs, filepath.Join(baseDir, filename), []byte("let foo='bar';"), 0o777), qt.IsNil) + } + + conf := testconfig.GetTestConfig(mfs, config.New()) + fs := hugofs.NewFrom(mfs, conf.BaseConfig()) + + p, err := paths.New(fs, conf) + c.Assert(err, qt.IsNil) + bfs, err := filesystems.NewBase(p, nil) + c.Assert(err, qt.IsNil) + resolver := newFSResolver(bfs.Assets.Fs) + + got := resolver.resolveComponent(test.impPath) + + gotPath := "" + expect := test.expect + if got != nil { + gotPath = filepath.ToSlash(got.Filename) + expect = path.Join(baseDir, test.expect) + } + + c.Assert(gotPath, qt.Equals, expect) + }) + } +} diff --git a/internal/js/esbuild/sourcemap.go b/internal/js/esbuild/sourcemap.go new file mode 100644 index 000000000..647f0c081 --- /dev/null +++ b/internal/js/esbuild/sourcemap.go @@ -0,0 +1,80 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package esbuild + +import ( + "encoding/json" + "strings" + + "github.com/evanw/esbuild/pkg/api" + "github.com/gohugoio/hugo/common/paths" +) + +type sourceMap struct { + Version int `json:"version"` + Sources []string `json:"sources"` + SourcesContent []string `json:"sourcesContent"` + Mappings string `json:"mappings"` + Names []string `json:"names"` +} + +func fixOutputFile(o *api.OutputFile, resolve func(string) string) error { + if strings.HasSuffix(o.Path, ".map") { + b, err := fixSourceMap(o.Contents, resolve) + if err != nil { + return err + } + o.Contents = b + } + return nil +} + +func fixSourceMap(s []byte, resolve func(string) string) ([]byte, error) { + var sm sourceMap + if err := json.Unmarshal([]byte(s), &sm); err != nil { + return nil, err + } + + sm.Sources = fixSourceMapSources(sm.Sources, resolve) + + b, err := json.Marshal(sm) + if err != nil { + return nil, err + } + + return b, nil +} + +func fixSourceMapSources(s []string, resolve func(string) string) []string { + var result []string + for _, src := range s { + if s := resolve(src); s != "" { + // Absolute filenames works fine on U*ix (tested in Chrome on MacOs), but works very poorly on Windows (again Chrome). + // So, convert it to a URL. + if u, err := paths.UrlFromFilename(s); err == nil { + result = append(result, u.String()) + } + } + } + return result +} + +// Used in tests. +func SourcesFromSourceMap(s string) []string { + var sm sourceMap + if err := json.Unmarshal([]byte(s), &sm); err != nil { + return nil + } + return sm.Sources +} diff --git a/lazy/init.go b/lazy/init.go index 7b88a5351..bef3867a9 100644 --- a/lazy/init.go +++ b/lazy/init.go @@ -36,7 +36,7 @@ type Init struct { prev *Init children []*Init - init onceMore + init OnceMore out any err error f func(context.Context) (any, error) diff --git a/lazy/once.go b/lazy/once.go index cea096652..dac689df3 100644 --- a/lazy/once.go +++ b/lazy/once.go @@ -24,13 +24,13 @@ import ( // * it can be reset, so the action can be repeated if needed // * it has methods to check if it's done or in progress -type onceMore struct { +type OnceMore struct { mu sync.Mutex lock uint32 done uint32 } -func (t *onceMore) Do(f func()) { +func (t *OnceMore) Do(f func()) { if atomic.LoadUint32(&t.done) == 1 { return } @@ -53,15 +53,15 @@ func (t *onceMore) Do(f func()) { f() } -func (t *onceMore) InProgress() bool { +func (t *OnceMore) InProgress() bool { return atomic.LoadUint32(&t.lock) == 1 } -func (t *onceMore) Done() bool { +func (t *OnceMore) Done() bool { return atomic.LoadUint32(&t.done) == 1 } -func (t *onceMore) ResetWithLock() *sync.Mutex { +func (t *OnceMore) ResetWithLock() *sync.Mutex { t.mu.Lock() defer atomic.StoreUint32(&t.done, 0) return &t.mu diff --git a/media/mediaType.go b/media/mediaType.go index a7ba1309a..97b10879c 100644 --- a/media/mediaType.go +++ b/media/mediaType.go @@ -273,9 +273,13 @@ func (t Types) GetByType(tp string) (Type, bool) { return Type{}, false } +func (t Types) normalizeSuffix(s string) string { + return strings.ToLower(strings.TrimPrefix(s, ".")) +} + // BySuffix will return all media types matching a suffix. func (t Types) BySuffix(suffix string) []Type { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) var types []Type for _, tt := range t { if tt.hasSuffix(suffix) { @@ -287,7 +291,7 @@ func (t Types) BySuffix(suffix string) []Type { // GetFirstBySuffix will return the first type matching the given suffix. func (t Types) GetFirstBySuffix(suffix string) (Type, SuffixInfo, bool) { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) for _, tt := range t { if tt.hasSuffix(suffix) { return tt, SuffixInfo{ @@ -304,7 +308,7 @@ func (t Types) GetFirstBySuffix(suffix string) (Type, SuffixInfo, bool) { // is ambiguous. // The lookup is case insensitive. func (t Types) GetBySuffix(suffix string) (tp Type, si SuffixInfo, found bool) { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) for _, tt := range t { if tt.hasSuffix(suffix) { if found { @@ -324,7 +328,7 @@ func (t Types) GetBySuffix(suffix string) (tp Type, si SuffixInfo, found bool) { } func (t Types) IsTextSuffix(suffix string) bool { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) for _, tt := range t { if tt.hasSuffix(suffix) { return tt.IsText() diff --git a/resources/image.go b/resources/image.go index 4595866d4..686f70e27 100644 --- a/resources/image.go +++ b/resources/image.go @@ -51,6 +51,7 @@ var ( _ resource.Source = (*imageResource)(nil) _ resource.Cloner = (*imageResource)(nil) _ resource.NameNormalizedProvider = (*imageResource)(nil) + _ targetPathProvider = (*imageResource)(nil) ) // imageResource represents an image resource. @@ -160,6 +161,10 @@ func (i *imageResource) Colors() ([]images.Color, error) { return i.dominantColors, nil } +func (i *imageResource) targetPath() string { + return i.TargetPath() +} + // Clone is for internal use. func (i *imageResource) Clone() resource.Resource { gr := i.baseResource.Clone().(baseResource) diff --git a/resources/page/page.go b/resources/page/page.go index 032ee320d..40e15dc6f 100644 --- a/resources/page/page.go +++ b/resources/page/page.go @@ -63,8 +63,7 @@ type ChildCareProvider interface { // section. RegularPagesRecursive() Pages - // Resources returns a list of all resources. - Resources() resource.Resources + resource.ResourcesProvider } type MarkupProvider interface { diff --git a/resources/resource.go b/resources/resource.go index cc7008e5a..6025cbf4c 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -47,6 +47,8 @@ var ( _ resource.Cloner = (*genericResource)(nil) _ resource.ResourcesLanguageMerger = (*resource.Resources)(nil) _ resource.Identifier = (*genericResource)(nil) + _ targetPathProvider = (*genericResource)(nil) + _ sourcePathProvider = (*genericResource)(nil) _ identity.IdentityGroupProvider = (*genericResource)(nil) _ identity.DependencyManagerProvider = (*genericResource)(nil) _ identity.Identity = (*genericResource)(nil) @@ -79,6 +81,7 @@ type ResourceSourceDescriptor struct { TargetPath string BasePathRelPermalink string BasePathTargetPath string + SourceFilenameOrPath string // Used for error logging. // The Data to associate with this resource. Data map[string]any @@ -463,6 +466,17 @@ func (l *genericResource) Key() string { return key } +func (l *genericResource) targetPath() string { + return l.paths.TargetPath() +} + +func (l *genericResource) sourcePath() string { + if p := l.sd.SourceFilenameOrPath; p != "" { + return p + } + return "" +} + func (l *genericResource) MediaType() media.Type { return l.sd.MediaType } @@ -660,3 +674,43 @@ func (r *resourceHash) init(l hugio.ReadSeekCloserProvider) error { func hashImage(r io.ReadSeeker) (uint64, int64, error) { return hashing.XXHashFromReader(r) } + +// InternalResourceTargetPath is used internally to get the target path for a Resource. +func InternalResourceTargetPath(r resource.Resource) string { + return r.(targetPathProvider).targetPath() +} + +// InternalResourceSourcePathBestEffort is used internally to get the source path for a Resource. +// It returns an empty string if the source path is not available. +func InternalResourceSourcePath(r resource.Resource) string { + if sp, ok := r.(sourcePathProvider); ok { + if p := sp.sourcePath(); p != "" { + return p + } + } + return "" +} + +// InternalResourceSourcePathBestEffort is used internally to get the source path for a Resource. +// Used for error messages etc. +// It will fall back to the target path if the source path is not available. +func InternalResourceSourcePathBestEffort(r resource.Resource) string { + if s := InternalResourceSourcePath(r); s != "" { + return s + } + return InternalResourceTargetPath(r) +} + +type targetPathProvider interface { + // targetPath is the relative path to this resource. + // In most cases this will be the same as the RelPermalink(), + // but it will not trigger any lazy publishing. + targetPath() string +} + +// Optional interface implemented by resources that can provide the source path. +type sourcePathProvider interface { + // sourcePath is the source path to this resource's source. + // This is used in error messages etc. + sourcePath() string +} diff --git a/resources/resource/resource_helpers.go b/resources/resource/resource_helpers.go index c2bb463c8..8575ae79e 100644 --- a/resources/resource/resource_helpers.go +++ b/resources/resource/resource_helpers.go @@ -14,10 +14,11 @@ package resource import ( - "github.com/gohugoio/hugo/common/maps" "strings" "time" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/helpers" "github.com/pelletier/go-toml/v2" diff --git a/resources/resource/resources.go b/resources/resource/resources.go index 32bcdbb08..480c703b5 100644 --- a/resources/resource/resources.go +++ b/resources/resource/resources.go @@ -16,8 +16,11 @@ package resource import ( "fmt" + "path" "strings" + "github.com/gohugoio/hugo/common/hreflect" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/hugofs/glob" "github.com/spf13/cast" @@ -29,6 +32,51 @@ var _ ResourceFinder = (*Resources)(nil) // I.e. both pages and images etc. type Resources []Resource +// Mount mounts the given resources from base to the given target path. +// Note that leading slashes in target marks an absolute path. +// This method is currently only useful in js.Batch. +func (r Resources) Mount(base, target string) ResourceGetter { + return resourceGetterFunc(func(namev any) Resource { + name1, err := cast.ToStringE(namev) + if err != nil { + panic(err) + } + + isTargetAbs := strings.HasPrefix(target, "/") + + if target != "" { + name1 = strings.TrimPrefix(name1, target) + if !isTargetAbs { + name1 = paths.TrimLeading(name1) + } + } + + if base != "" && isTargetAbs { + name1 = path.Join(base, name1) + } + + for _, res := range r { + name2 := res.Name() + + if base != "" && !isTargetAbs { + name2 = paths.TrimLeading(strings.TrimPrefix(name2, base)) + } + + if strings.EqualFold(name1, name2) { + return res + } + + } + + return nil + }) +} + +type ResourcesProvider interface { + // Resources returns a list of all resources. + Resources() Resources +} + // var _ resource.ResourceFinder = (*Namespace)(nil) // ResourcesConverter converts a given slice of Resource objects to Resources. type ResourcesConverter interface { @@ -63,13 +111,25 @@ func (r Resources) Get(name any) Resource { panic(err) } - namestr = paths.AddLeadingSlash(namestr) + isDotCurrent := strings.HasPrefix(namestr, "./") + if isDotCurrent { + namestr = strings.TrimPrefix(namestr, "./") + } else { + namestr = paths.AddLeadingSlash(namestr) + } + + check := func(name string) bool { + if !isDotCurrent { + name = paths.AddLeadingSlash(name) + } + return strings.EqualFold(namestr, name) + } // First check the Name. // Note that this can be modified by the user in the front matter, // also, it does not contain any language code. for _, resource := range r { - if strings.EqualFold(namestr, paths.AddLeadingSlash(resource.Name())) { + if check(resource.Name()) { return resource } } @@ -77,7 +137,7 @@ func (r Resources) Get(name any) Resource { // Finally, check the normalized name. for _, resource := range r { if nop, ok := resource.(NameNormalizedProvider); ok { - if strings.EqualFold(namestr, paths.AddLeadingSlash(nop.NameNormalized())) { + if check(nop.NameNormalized()) { return resource } } @@ -197,14 +257,35 @@ type Source interface { Publish() error } -// ResourceFinder provides methods to find Resources. -// Note that GetRemote (as found in resources.GetRemote) is -// not covered by this interface, as this is only available as a global template function. -type ResourceFinder interface { +type ResourceGetter interface { // Get locates the Resource with the given name in the current context (e.g. in .Page.Resources). // // It returns nil if no Resource could found, panics if name is invalid. Get(name any) Resource +} + +type IsProbablySameResourceGetter interface { + IsProbablySameResourceGetter(other ResourceGetter) bool +} + +// StaleInfoResourceGetter is a ResourceGetter that also provides information about +// whether the underlying resources are stale. +type StaleInfoResourceGetter interface { + StaleInfo + ResourceGetter +} + +type resourceGetterFunc func(name any) Resource + +func (f resourceGetterFunc) Get(name any) Resource { + return f(name) +} + +// ResourceFinder provides methods to find Resources. +// Note that GetRemote (as found in resources.GetRemote) is +// not covered by this interface, as this is only available as a global template function. +type ResourceFinder interface { + ResourceGetter // GetMatch finds the first Resource matching the given pattern, or nil if none found. // @@ -235,3 +316,92 @@ type ResourceFinder interface { // It returns nil if no Resources could found, panics if typ is invalid. ByType(typ any) Resources } + +// NewCachedResourceGetter creates a new ResourceGetter from the given objects. +// If multiple objects are provided, they are merged into one where +// the first match wins. +func NewCachedResourceGetter(os ...any) *cachedResourceGetter { + var getters multiResourceGetter + for _, o := range os { + if g, ok := unwrapResourceGetter(o); ok { + getters = append(getters, g) + } + } + + return &cachedResourceGetter{ + cache: maps.NewCache[string, Resource](), + delegate: getters, + } +} + +type multiResourceGetter []ResourceGetter + +func (m multiResourceGetter) Get(name any) Resource { + for _, g := range m { + if res := g.Get(name); res != nil { + return res + } + } + return nil +} + +var ( + _ ResourceGetter = (*cachedResourceGetter)(nil) + _ IsProbablySameResourceGetter = (*cachedResourceGetter)(nil) +) + +type cachedResourceGetter struct { + cache *maps.Cache[string, Resource] + delegate ResourceGetter +} + +func (c *cachedResourceGetter) Get(name any) Resource { + namestr, err := cast.ToStringE(name) + if err != nil { + panic(err) + } + v, _ := c.cache.GetOrCreate(namestr, func() (Resource, error) { + v := c.delegate.Get(name) + return v, nil + }) + return v +} + +func (c *cachedResourceGetter) IsProbablySameResourceGetter(other ResourceGetter) bool { + isProbablyEq := true + c.cache.ForEeach(func(k string, v Resource) bool { + if v != other.Get(k) { + isProbablyEq = false + return false + } + return true + }) + + return isProbablyEq +} + +func unwrapResourceGetter(v any) (ResourceGetter, bool) { + if v == nil { + return nil, false + } + switch vv := v.(type) { + case ResourceGetter: + return vv, true + case ResourcesProvider: + return vv.Resources(), true + case func(name any) Resource: + return resourceGetterFunc(vv), true + default: + vvv, ok := hreflect.ToSliceAny(v) + if !ok { + return nil, false + } + var getters multiResourceGetter + for _, vv := range vvv { + if g, ok := unwrapResourceGetter(vv); ok { + getters = append(getters, g) + } + } + return getters, len(getters) > 0 + } +} diff --git a/resources/resource/resources_integration_test.go b/resources/resource/resources_integration_test.go new file mode 100644 index 000000000..920fc7397 --- /dev/null +++ b/resources/resource/resources_integration_test.go @@ -0,0 +1,105 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource_test + +import ( + "testing" + + "github.com/gohugoio/hugo/hugolib" +) + +func TestResourcesMount(t *testing.T) { + files := ` +-- hugo.toml -- +-- assets/text/txt1.txt -- +Text 1. +-- assets/text/txt2.txt -- +Text 2. +-- assets/text/sub/txt3.txt -- +Text 3. +-- assets/text/sub/txt4.txt -- +Text 4. +-- content/mybundle/index.md -- +--- +title: "My Bundle" +--- +-- content/mybundle/txt1.txt -- +Text 1. +-- content/mybundle/sub/txt2.txt -- +Text 1. +-- layouts/index.html -- +{{ $mybundle := site.GetPage "mybundle" }} +{{ $subResources := resources.Match "/text/sub/*.*" }} +{{ $subResourcesMount := $subResources.Mount "/text/sub" "/newroot" }} +resources:text/txt1.txt:{{ with resources.Get "text/txt1.txt" }}{{ .Name }}{{ end }}| +resources:text/txt2.txt:{{ with resources.Get "text/txt2.txt" }}{{ .Name }}{{ end }}| +resources:text/sub/txt3.txt:{{ with resources.Get "text/sub/txt3.txt" }}{{ .Name }}{{ end }}| +subResources.range:{{ range $subResources }}{{ .Name }}|{{ end }}| +subResources:"text/sub/txt3.txt:{{ with $subResources.Get "text/sub/txt3.txt" }}{{ .Name }}{{ end }}| +subResourcesMount:/newroot/txt3.txt:{{ with $subResourcesMount.Get "/newroot/txt3.txt" }}{{ .Name }}{{ end }}| +page:txt1.txt:{{ with $mybundle.Resources.Get "txt1.txt" }}{{ .Name }}{{ end }}| +page:./txt1.txt:{{ with $mybundle.Resources.Get "./txt1.txt" }}{{ .Name }}{{ end }}| +page:sub/txt2.txt:{{ with $mybundle.Resources.Get "sub/txt2.txt" }}{{ .Name }}{{ end }}| +` + b := hugolib.Test(t, files) + + b.AssertFileContent("public/index.html", ` +resources:text/txt1.txt:/text/txt1.txt| +resources:text/txt2.txt:/text/txt2.txt| +resources:text/sub/txt3.txt:/text/sub/txt3.txt| +subResources:"text/sub/txt3.txt:/text/sub/txt3.txt| +subResourcesMount:/newroot/txt3.txt:/text/sub/txt3.txt| +page:txt1.txt:txt1.txt| +page:./txt1.txt:txt1.txt| +page:sub/txt2.txt:sub/txt2.txt| +`) +} + +func TestResourcesMountOnRename(t *testing.T) { + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "home", "sitemap"] +-- content/mybundle/index.md -- +--- +title: "My Bundle" +resources: +- name: /foo/bars.txt + src: foo/txt1.txt +- name: foo/bars2.txt + src: foo/txt2.txt +--- +-- content/mybundle/foo/txt1.txt -- +Text 1. +-- content/mybundle/foo/txt2.txt -- +Text 2. +-- layouts/_default/single.html -- +Single. +{{ $mybundle := site.GetPage "mybundle" }} +Resources:{{ range $mybundle.Resources }}Name: {{ .Name }}|{{ end }}$ +{{ $subResourcesMount := $mybundle.Resources.Mount "/foo" "/newroot" }} + {{ $subResourcesMount2 := $mybundle.Resources.Mount "foo" "/newroot" }} +{{ $subResourcesMount3 := $mybundle.Resources.Mount "foo" "." }} +subResourcesMount:/newroot/bars.txt:{{ with $subResourcesMount.Get "/newroot/bars.txt" }}{{ .Name }}{{ end }}| +subResourcesMount:/newroot/bars2.txt:{{ with $subResourcesMount.Get "/newroot/bars2.txt" }}{{ .Name }}{{ end }}| +subResourcesMount2:/newroot/bars2.txt:{{ with $subResourcesMount2.Get "/newroot/bars2.txt" }}{{ .Name }}{{ end }}| +subResourcesMount3:bars2.txt:{{ with $subResourcesMount3.Get "bars2.txt" }}{{ .Name }}{{ end }}| +` + b := hugolib.Test(t, files) + b.AssertFileContent("public/mybundle/index.html", + "Resources:Name: foo/bars.txt|Name: foo/bars2.txt|$", + "subResourcesMount:/newroot/bars.txt:|\nsubResourcesMount:/newroot/bars2.txt:|", + "subResourcesMount2:/newroot/bars2.txt:foo/bars2.txt|", + "subResourcesMount3:bars2.txt:foo/bars2.txt|", + ) +} diff --git a/resources/resource/resources_test.go b/resources/resource/resources_test.go new file mode 100644 index 000000000..ebadbb312 --- /dev/null +++ b/resources/resource/resources_test.go @@ -0,0 +1,122 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestResourcesMount(t *testing.T) { + c := qt.New(t) + c.Assert(true, qt.IsTrue) + + var m ResourceGetter + var r Resources + + check := func(in, expect string) { + c.Helper() + r := m.Get(in) + c.Assert(r, qt.Not(qt.IsNil)) + c.Assert(r.Name(), qt.Equals, expect) + } + + checkNil := func(in string) { + c.Helper() + r := m.Get(in) + c.Assert(r, qt.IsNil) + } + + // Misc tests. + r = Resources{ + testResource{name: "/foo/theme.css"}, + } + + m = r.Mount("/foo", ".") + check("./theme.css", "/foo/theme.css") + + // Relative target. + r = Resources{ + testResource{name: "/a/b/c/d.txt"}, + testResource{name: "/a/b/c/e/f.txt"}, + testResource{name: "/a/b/d.txt"}, + testResource{name: "/a/b/e.txt"}, + } + + m = r.Mount("/a/b/c", "z") + check("z/d.txt", "/a/b/c/d.txt") + check("z/e/f.txt", "/a/b/c/e/f.txt") + + m = r.Mount("/a/b", "") + check("d.txt", "/a/b/d.txt") + m = r.Mount("/a/b", ".") + check("d.txt", "/a/b/d.txt") + m = r.Mount("/a/b", "./") + check("d.txt", "/a/b/d.txt") + check("./d.txt", "/a/b/d.txt") + + m = r.Mount("/a/b", ".") + check("./d.txt", "/a/b/d.txt") + + // Absolute target. + m = r.Mount("/a/b/c", "/z") + check("/z/d.txt", "/a/b/c/d.txt") + check("/z/e/f.txt", "/a/b/c/e/f.txt") + checkNil("/z/f.txt") + + m = r.Mount("/a/b", "/z") + check("/z/c/d.txt", "/a/b/c/d.txt") + check("/z/c/e/f.txt", "/a/b/c/e/f.txt") + check("/z/d.txt", "/a/b/d.txt") + checkNil("/z/f.txt") + + m = r.Mount("", "") + check("/a/b/c/d.txt", "/a/b/c/d.txt") + check("/a/b/c/e/f.txt", "/a/b/c/e/f.txt") + check("/a/b/d.txt", "/a/b/d.txt") + checkNil("/a/b/f.txt") + + m = r.Mount("/a/b", "/a/b") + check("/a/b/c/d.txt", "/a/b/c/d.txt") + check("/a/b/c/e/f.txt", "/a/b/c/e/f.txt") + check("/a/b/d.txt", "/a/b/d.txt") + checkNil("/a/b/f.txt") + + // Resources with relative paths. + r = Resources{ + testResource{name: "a/b/c/d.txt"}, + testResource{name: "a/b/c/e/f.txt"}, + testResource{name: "a/b/d.txt"}, + testResource{name: "a/b/e.txt"}, + testResource{name: "n.txt"}, + } + + m = r.Mount("a/b", "z") + check("z/d.txt", "a/b/d.txt") + checkNil("/z/d.txt") +} + +type testResource struct { + Resource + name string +} + +func (r testResource) Name() string { + return r.name +} + +func (r testResource) NameNormalized() string { + return r.name +} diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 2d868bd15..7dd26f4c0 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -143,16 +143,18 @@ func (c *Client) Get(pathname string) (resource.Resource, error) { return nil, err } - pi := fi.(hugofs.FileMetaInfo).Meta().PathInfo + meta := fi.(hugofs.FileMetaInfo).Meta() + pi := meta.PathInfo return c.rs.NewResource(resources.ResourceSourceDescriptor{ LazyPublish: true, OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { return c.rs.BaseFs.Assets.Fs.Open(filename) }, - Path: pi, - GroupIdentity: pi, - TargetPath: pathname, + Path: pi, + GroupIdentity: pi, + TargetPath: pathname, + SourceFilenameOrPath: meta.Filename, }) }) } @@ -196,10 +198,11 @@ func (c *Client) match(name, pattern string, matchFunc func(r resource.Resource) OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { return meta.Open() }, - NameNormalized: meta.PathInfo.Path(), - NameOriginal: meta.PathInfo.Unnormalized().Path(), - GroupIdentity: meta.PathInfo, - TargetPath: meta.PathInfo.Unnormalized().Path(), + NameNormalized: meta.PathInfo.Path(), + NameOriginal: meta.PathInfo.Unnormalized().Path(), + GroupIdentity: meta.PathInfo, + TargetPath: meta.PathInfo.Unnormalized().Path(), + SourceFilenameOrPath: meta.Filename, }) if err != nil { return true, err diff --git a/resources/resource_metadata.go b/resources/resource_metadata.go index 7d2459225..8861ded5c 100644 --- a/resources/resource_metadata.go +++ b/resources/resource_metadata.go @@ -15,6 +15,7 @@ package resources import ( "fmt" + "path/filepath" "strconv" "strings" @@ -26,6 +27,7 @@ import ( "github.com/spf13/cast" "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/paths" ) var ( @@ -172,6 +174,8 @@ func assignMetadata(metadata []map[string]any, ma *metaResource) error { name, found := meta["name"] if found { name := cast.ToString(name) + // Bundled resources in sub folders are relative paths with forward slashes. Make sure any renames also matches that format: + name = paths.TrimLeading(filepath.ToSlash(name)) if !nameCounterFound { nameCounterFound = strings.Contains(name, counterPlaceHolder) } diff --git a/resources/resource_test.go b/resources/resource_test.go index ccce8eff7..d07770456 100644 --- a/resources/resource_test.go +++ b/resources/resource_test.go @@ -16,8 +16,26 @@ package resources import ( "os" "testing" + + qt "github.com/frankban/quicktest" ) +func TestAtomicStaler(t *testing.T) { + c := qt.New(t) + + type test struct { + AtomicStaler + } + + var v test + + c.Assert(v.StaleVersion(), qt.Equals, uint32(0)) + v.MarkStale() + c.Assert(v.StaleVersion(), qt.Equals, uint32(1)) + v.MarkStale() + c.Assert(v.StaleVersion(), qt.Equals, uint32(2)) +} + func BenchmarkHashImage(b *testing.B) { f, err := os.Open("testdata/sunset.jpg") if err != nil { diff --git a/resources/resource_transformers/js/build.go b/resources/resource_transformers/js/build.go index cc68d2253..bd943461f 100644 --- a/resources/resource_transformers/js/build.go +++ b/resources/resource_transformers/js/build.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,209 +14,69 @@ package js import ( - "errors" - "fmt" - "io" - "os" "path" - "path/filepath" "regexp" - "strings" - - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/media" - - "github.com/gohugoio/hugo/common/herrors" - "github.com/gohugoio/hugo/common/text" + "github.com/evanw/esbuild/pkg/api" "github.com/gohugoio/hugo/hugolib/filesystems" - "github.com/gohugoio/hugo/resources/internal" + "github.com/gohugoio/hugo/internal/js/esbuild" - "github.com/evanw/esbuild/pkg/api" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" ) // Client context for ESBuild. type Client struct { - rs *resources.Spec - sfs *filesystems.SourceFilesystem + c *esbuild.BuildClient } // New creates a new client context. func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) *Client { return &Client{ - rs: rs, - sfs: fs, + c: esbuild.NewBuildClient(fs, rs), } } -type buildTransformation struct { - optsm map[string]any - c *Client -} - -func (t *buildTransformation) Key() internal.ResourceTransformationKey { - return internal.NewResourceTransformationKey("jsbuild", t.optsm) +// Process processes a resource with the user provided options. +func (c *Client) Process(res resources.ResourceTransformer, opts map[string]any) (resource.Resource, error) { + return res.Transform( + &buildTransformation{c: c, optsm: opts}, + ) } -func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { - ctx.OutMediaType = media.Builtin.JavascriptType - - opts, err := decodeOptions(t.optsm) - if err != nil { - return err +func (c *Client) transform(opts esbuild.Options, transformCtx *resources.ResourceTransformationCtx) (api.BuildResult, error) { + if transformCtx.DependencyManager != nil { + opts.DependencyManager = transformCtx.DependencyManager } - if opts.TargetPath != "" { - ctx.OutPath = opts.TargetPath - } else { - ctx.ReplaceOutPathExtension(".js") - } + opts.StdinSourcePath = transformCtx.SourcePath - src, err := io.ReadAll(ctx.From) + result, err := c.c.Build(opts) if err != nil { - return err + return result, err } - opts.sourceDir = filepath.FromSlash(path.Dir(ctx.SourcePath)) - opts.resolveDir = t.c.rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved - opts.contents = string(src) - opts.mediaType = ctx.InMediaType - opts.tsConfig = t.c.rs.ResolveJSConfigFile("tsconfig.json") - - buildOptions, err := toBuildOptions(opts) - if err != nil { - return err - } - - buildOptions.Plugins, err = createBuildPlugins(ctx.DependencyManager, t.c, opts) - if err != nil { - return err - } - - if buildOptions.Sourcemap == api.SourceMapExternal && buildOptions.Outdir == "" { - buildOptions.Outdir, err = os.MkdirTemp(os.TempDir(), "compileOutput") - if err != nil { - return err - } - defer os.Remove(buildOptions.Outdir) - } - - if opts.Inject != nil { - // Resolve the absolute filenames. - for i, ext := range opts.Inject { - impPath := filepath.FromSlash(ext) - if filepath.IsAbs(impPath) { - return fmt.Errorf("inject: absolute paths not supported, must be relative to /assets") - } - - m := resolveComponentInAssets(t.c.rs.Assets.Fs, impPath) - - if m == nil { - return fmt.Errorf("inject: file %q not found", ext) - } - - opts.Inject[i] = m.Filename - - } - - buildOptions.Inject = opts.Inject - - } - - result := api.Build(buildOptions) - - if len(result.Errors) > 0 { - - createErr := func(msg api.Message) error { - loc := msg.Location - if loc == nil { - return errors.New(msg.Text) - } - path := loc.File - if path == stdinImporter { - path = ctx.SourcePath - } - - errorMessage := msg.Text - errorMessage = strings.ReplaceAll(errorMessage, nsImportHugo+":", "") - - var ( - f afero.File - err error - ) - - if strings.HasPrefix(path, nsImportHugo) { - path = strings.TrimPrefix(path, nsImportHugo+":") - f, err = hugofs.Os.Open(path) - } else { - var fi os.FileInfo - fi, err = t.c.sfs.Fs.Stat(path) - if err == nil { - m := fi.(hugofs.FileMetaInfo).Meta() - path = m.Filename - f, err = m.Open() - } - - } - - if err == nil { - fe := herrors. - NewFileErrorFromName(errors.New(errorMessage), path). - UpdatePosition(text.Position{Offset: -1, LineNumber: loc.Line, ColumnNumber: loc.Column}). - UpdateContent(f, nil) - - f.Close() - return fe - } - - return fmt.Errorf("%s", errorMessage) - } - - var errors []error - - for _, msg := range result.Errors { - errors = append(errors, createErr(msg)) - } - - // Return 1, log the rest. - for i, err := range errors { - if i > 0 { - t.c.rs.Logger.Errorf("js.Build failed: %s", err) - } - } - - return errors[0] - } - - if buildOptions.Sourcemap == api.SourceMapExternal { + if opts.ExternalOptions.SourceMap == "linked" || opts.ExternalOptions.SourceMap == "external" { content := string(result.OutputFiles[1].Contents) - symPath := path.Base(ctx.OutPath) + ".map" - re := regexp.MustCompile(`//# sourceMappingURL=.*\n?`) - content = re.ReplaceAllString(content, "//# sourceMappingURL="+symPath+"\n") + if opts.ExternalOptions.SourceMap == "linked" { + symPath := path.Base(transformCtx.OutPath) + ".map" + re := regexp.MustCompile(`//# sourceMappingURL=.*\n?`) + content = re.ReplaceAllString(content, "//# sourceMappingURL="+symPath+"\n") + } - if err = ctx.PublishSourceMap(string(result.OutputFiles[0].Contents)); err != nil { - return err + if err = transformCtx.PublishSourceMap(string(result.OutputFiles[0].Contents)); err != nil { + return result, err } - _, err := ctx.To.Write([]byte(content)) + _, err := transformCtx.To.Write([]byte(content)) if err != nil { - return err + return result, err } } else { - _, err := ctx.To.Write(result.OutputFiles[0].Contents) + _, err := transformCtx.To.Write(result.OutputFiles[0].Contents) if err != nil { - return err + return result, err } - } - return nil -} -// Process process esbuild transform -func (c *Client) Process(res resources.ResourceTransformer, opts map[string]any) (resource.Resource, error) { - return res.Transform( - &buildTransformation{c: c, optsm: opts}, - ) + } + return result, nil } diff --git a/resources/resource_transformers/js/js_integration_test.go b/resources/resource_transformers/js/js_integration_test.go index 304c51d33..c62312ef5 100644 --- a/resources/resource_transformers/js/js_integration_test.go +++ b/resources/resource_transformers/js/js_integration_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,13 +14,16 @@ package js_test import ( + "os" "path/filepath" "strings" "testing" qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/hugolib" + "github.com/gohugoio/hugo/internal/js/esbuild" ) func TestBuildVariants(t *testing.T) { @@ -173,7 +176,7 @@ hello: hello: other: "Bonjour" -- layouts/index.html -- -{{ $options := dict "minify" false "externals" (slice "react" "react-dom") }} +{{ $options := dict "minify" false "externals" (slice "react" "react-dom") "sourcemap" "linked" }} {{ $js := resources.Get "js/main.js" | js.Build $options }} JS: {{ template "print" $js }} {{ $jsx := resources.Get "js/myjsx.jsx" | js.Build $options }} @@ -201,14 +204,31 @@ TS2: {{ template "print" $ts2 }} TxtarString: files, }).Build() - b.AssertFileContent("public/js/myts.js", `//# sourceMappingURL=data:application/json;base64,ewogICJ2ZXJz`) - b.AssertFileContent("public/js/myts2.js.map", `"version": 3,`) + b.AssertFileContent("public/js/main.js", `//# sourceMappingURL=main.js.map`) + b.AssertFileContent("public/js/main.js.map", `"version":3`, "! ns-hugo") // linked + b.AssertFileContent("public/js/myts.js", `//# sourceMappingURL=data:application/json;base64,ewogICJ2ZXJz`) // inline b.AssertFileContent("public/index.html", ` console.log("included"); if (hasSpace.test(string)) var React = __toESM(__require("react")); function greeter(person) { `) + + checkMap := func(p string, expectLen int) { + s := b.FileContent(p) + sources := esbuild.SourcesFromSourceMap(s) + b.Assert(sources, qt.HasLen, expectLen) + + // Check that all source files exist. + for _, src := range sources { + filename, ok := paths.UrlStringToFilename(src) + b.Assert(ok, qt.IsTrue) + _, err := os.Stat(filename) + b.Assert(err, qt.IsNil, qt.Commentf("src: %q", src)) + } + } + + checkMap("public/js/main.js.map", 4) } func TestBuildError(t *testing.T) { diff --git a/resources/resource_transformers/js/options.go b/resources/resource_transformers/js/options.go deleted file mode 100644 index 8c271d032..000000000 --- a/resources/resource_transformers/js/options.go +++ /dev/null @@ -1,461 +0,0 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package js - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/gohugoio/hugo/common/maps" - "github.com/gohugoio/hugo/common/paths" - "github.com/gohugoio/hugo/identity" - "github.com/spf13/afero" - - "github.com/evanw/esbuild/pkg/api" - - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/media" - "github.com/mitchellh/mapstructure" -) - -const ( - nsImportHugo = "ns-hugo" - nsParams = "ns-params" - - stdinImporter = "<stdin>" -) - -// Options esbuild configuration -type Options struct { - // If not set, the source path will be used as the base target path. - // Note that the target path's extension may change if the target MIME type - // is different, e.g. when the source is TypeScript. - TargetPath string - - // Whether to minify to output. - Minify bool - - // Whether to write mapfiles - SourceMap string - - // The language target. - // One of: es2015, es2016, es2017, es2018, es2019, es2020 or esnext. - // Default is esnext. - Target string - - // The output format. - // One of: iife, cjs, esm - // Default is to esm. - Format string - - // External dependencies, e.g. "react". - Externals []string - - // This option allows you to automatically replace a global variable with an import from another file. - // The filenames must be relative to /assets. - // See https://esbuild.github.io/api/#inject - Inject []string - - // User defined symbols. - Defines map[string]any - - // Maps a component import to another. - Shims map[string]string - - // User defined params. Will be marshaled to JSON and available as "@params", e.g. - // import * as params from '@params'; - Params any - - // What to use instead of React.createElement. - JSXFactory string - - // What to use instead of React.Fragment. - JSXFragment string - - // What to do about JSX syntax. - // See https://esbuild.github.io/api/#jsx - JSX string - - // Which library to use to automatically import JSX helper functions from. Only works if JSX is set to automatic. - // See https://esbuild.github.io/api/#jsx-import-source - JSXImportSource string - - // There is/was a bug in WebKit with severe performance issue with the tracking - // of TDZ checks in JavaScriptCore. - // - // Enabling this flag removes the TDZ and `const` assignment checks and - // may improve performance of larger JS codebases until the WebKit fix - // is in widespread use. - // - // See https://bugs.webkit.org/show_bug.cgi?id=199866 - // Deprecated: This no longer have any effect and will be removed. - // TODO(bep) remove. See https://github.com/evanw/esbuild/commit/869e8117b499ca1dbfc5b3021938a53ffe934dba - AvoidTDZ bool - - mediaType media.Type - outDir string - contents string - sourceDir string - resolveDir string - tsConfig string -} - -func decodeOptions(m map[string]any) (Options, error) { - var opts Options - - if err := mapstructure.WeakDecode(m, &opts); err != nil { - return opts, err - } - - if opts.TargetPath != "" { - opts.TargetPath = paths.ToSlashTrimLeading(opts.TargetPath) - } - - opts.Target = strings.ToLower(opts.Target) - opts.Format = strings.ToLower(opts.Format) - - return opts, nil -} - -var extensionToLoaderMap = map[string]api.Loader{ - ".js": api.LoaderJS, - ".mjs": api.LoaderJS, - ".cjs": api.LoaderJS, - ".jsx": api.LoaderJSX, - ".ts": api.LoaderTS, - ".tsx": api.LoaderTSX, - ".css": api.LoaderCSS, - ".json": api.LoaderJSON, - ".txt": api.LoaderText, -} - -func loaderFromFilename(filename string) api.Loader { - l, found := extensionToLoaderMap[filepath.Ext(filename)] - if found { - return l - } - return api.LoaderJS -} - -func resolveComponentInAssets(fs afero.Fs, impPath string) *hugofs.FileMeta { - findFirst := func(base string) *hugofs.FileMeta { - // This is the most common sub-set of ESBuild's default extensions. - // We assume that imports of JSON, CSS etc. will be using their full - // name with extension. - for _, ext := range []string{".js", ".ts", ".tsx", ".jsx"} { - if strings.HasSuffix(impPath, ext) { - // Import of foo.js.js need the full name. - continue - } - if fi, err := fs.Stat(base + ext); err == nil { - return fi.(hugofs.FileMetaInfo).Meta() - } - } - - // Not found. - return nil - } - - var m *hugofs.FileMeta - - // We need to check if this is a regular file imported without an extension. - // There may be ambiguous situations where both foo.js and foo/index.js exists. - // This import order is in line with both how Node and ESBuild's native - // import resolver works. - - // It may be a regular file imported without an extension, e.g. - // foo or foo/index. - m = findFirst(impPath) - if m != nil { - return m - } - - base := filepath.Base(impPath) - if base == "index" { - // try index.esm.js etc. - m = findFirst(impPath + ".esm") - if m != nil { - return m - } - } - - // Check the path as is. - fi, err := fs.Stat(impPath) - - if err == nil { - if fi.IsDir() { - m = findFirst(filepath.Join(impPath, "index")) - if m == nil { - m = findFirst(filepath.Join(impPath, "index.esm")) - } - } else { - m = fi.(hugofs.FileMetaInfo).Meta() - } - } else if strings.HasSuffix(base, ".js") { - m = findFirst(strings.TrimSuffix(impPath, ".js")) - } - - return m -} - -func createBuildPlugins(depsManager identity.Manager, c *Client, opts Options) ([]api.Plugin, error) { - fs := c.rs.Assets - - resolveImport := func(args api.OnResolveArgs) (api.OnResolveResult, error) { - impPath := args.Path - if opts.Shims != nil { - override, found := opts.Shims[impPath] - if found { - impPath = override - } - } - isStdin := args.Importer == stdinImporter - var relDir string - if !isStdin { - rel, found := fs.MakePathRelative(args.Importer, true) - if !found { - // Not in any of the /assets folders. - // This is an import from a node_modules, let - // ESBuild resolve this. - return api.OnResolveResult{}, nil - } - - relDir = filepath.Dir(rel) - } else { - relDir = opts.sourceDir - } - - // Imports not starting with a "." is assumed to live relative to /assets. - // Hugo makes no assumptions about the directory structure below /assets. - if relDir != "" && strings.HasPrefix(impPath, ".") { - impPath = filepath.Join(relDir, impPath) - } - - m := resolveComponentInAssets(fs.Fs, impPath) - - if m != nil { - depsManager.AddIdentity(m.PathInfo) - - // Store the source root so we can create a jsconfig.json - // to help IntelliSense when the build is done. - // This should be a small number of elements, and when - // in server mode, we may get stale entries on renames etc., - // but that shouldn't matter too much. - c.rs.JSConfigBuilder.AddSourceRoot(m.SourceRoot) - return api.OnResolveResult{Path: m.Filename, Namespace: nsImportHugo}, nil - } - - // Fall back to ESBuild's resolve. - return api.OnResolveResult{}, nil - } - - importResolver := api.Plugin{ - Name: "hugo-import-resolver", - Setup: func(build api.PluginBuild) { - build.OnResolve(api.OnResolveOptions{Filter: `.*`}, - func(args api.OnResolveArgs) (api.OnResolveResult, error) { - return resolveImport(args) - }) - build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: nsImportHugo}, - func(args api.OnLoadArgs) (api.OnLoadResult, error) { - b, err := os.ReadFile(args.Path) - if err != nil { - return api.OnLoadResult{}, fmt.Errorf("failed to read %q: %w", args.Path, err) - } - c := string(b) - return api.OnLoadResult{ - // See https://github.com/evanw/esbuild/issues/502 - // This allows all modules to resolve dependencies - // in the main project's node_modules. - ResolveDir: opts.resolveDir, - Contents: &c, - Loader: loaderFromFilename(args.Path), - }, nil - }) - }, - } - - params := opts.Params - if params == nil { - // This way @params will always resolve to something. - params = make(map[string]any) - } - - b, err := json.Marshal(params) - if err != nil { - return nil, fmt.Errorf("failed to marshal params: %w", err) - } - bs := string(b) - paramsPlugin := api.Plugin{ - Name: "hugo-params-plugin", - Setup: func(build api.PluginBuild) { - build.OnResolve(api.OnResolveOptions{Filter: `^@params$`}, - func(args api.OnResolveArgs) (api.OnResolveResult, error) { - return api.OnResolveResult{ - Path: args.Path, - Namespace: nsParams, - }, nil - }) - build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: nsParams}, - func(args api.OnLoadArgs) (api.OnLoadResult, error) { - return api.OnLoadResult{ - Contents: &bs, - Loader: api.LoaderJSON, - }, nil - }) - }, - } - - return []api.Plugin{importResolver, paramsPlugin}, nil -} - -func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { - var target api.Target - switch opts.Target { - case "", "esnext": - target = api.ESNext - case "es5": - target = api.ES5 - case "es6", "es2015": - target = api.ES2015 - case "es2016": - target = api.ES2016 - case "es2017": - target = api.ES2017 - case "es2018": - target = api.ES2018 - case "es2019": - target = api.ES2019 - case "es2020": - target = api.ES2020 - case "es2021": - target = api.ES2021 - case "es2022": - target = api.ES2022 - case "es2023": - target = api.ES2023 - default: - err = fmt.Errorf("invalid target: %q", opts.Target) - return - } - - mediaType := opts.mediaType - if mediaType.IsZero() { - mediaType = media.Builtin.JavascriptType - } - - var loader api.Loader - switch mediaType.SubType { - // TODO(bep) ESBuild support a set of other loaders, but I currently fail - // to see the relevance. That may change as we start using this. - case media.Builtin.JavascriptType.SubType: - loader = api.LoaderJS - case media.Builtin.TypeScriptType.SubType: - loader = api.LoaderTS - case media.Builtin.TSXType.SubType: - loader = api.LoaderTSX - case media.Builtin.JSXType.SubType: - loader = api.LoaderJSX - default: - err = fmt.Errorf("unsupported Media Type: %q", opts.mediaType) - return - } - - var format api.Format - // One of: iife, cjs, esm - switch opts.Format { - case "", "iife": - format = api.FormatIIFE - case "esm": - format = api.FormatESModule - case "cjs": - format = api.FormatCommonJS - default: - err = fmt.Errorf("unsupported script output format: %q", opts.Format) - return - } - - var jsx api.JSX - switch opts.JSX { - case "", "transform": - jsx = api.JSXTransform - case "preserve": - jsx = api.JSXPreserve - case "automatic": - jsx = api.JSXAutomatic - default: - err = fmt.Errorf("unsupported jsx type: %q", opts.JSX) - return - } - - var defines map[string]string - if opts.Defines != nil { - defines = maps.ToStringMapString(opts.Defines) - } - - // By default we only need to specify outDir and no outFile - outDir := opts.outDir - outFile := "" - var sourceMap api.SourceMap - switch opts.SourceMap { - case "inline": - sourceMap = api.SourceMapInline - case "external": - sourceMap = api.SourceMapExternal - case "": - sourceMap = api.SourceMapNone - default: - err = fmt.Errorf("unsupported sourcemap type: %q", opts.SourceMap) - return - } - - buildOptions = api.BuildOptions{ - Outfile: outFile, - Bundle: true, - - Target: target, - Format: format, - Sourcemap: sourceMap, - - MinifyWhitespace: opts.Minify, - MinifyIdentifiers: opts.Minify, - MinifySyntax: opts.Minify, - - Outdir: outDir, - Define: defines, - - External: opts.Externals, - - JSXFactory: opts.JSXFactory, - JSXFragment: opts.JSXFragment, - - JSX: jsx, - JSXImportSource: opts.JSXImportSource, - - Tsconfig: opts.tsConfig, - - // Note: We're not passing Sourcefile to ESBuild. - // This makes ESBuild pass `stdin` as the Importer to the import - // resolver, which is what we need/expect. - Stdin: &api.StdinOptions{ - Contents: opts.contents, - ResolveDir: opts.resolveDir, - Loader: loader, - }, - } - return -} diff --git a/resources/resource_transformers/js/options_test.go b/resources/resource_transformers/js/options_test.go deleted file mode 100644 index 53aa9b6bb..000000000 --- a/resources/resource_transformers/js/options_test.go +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package js - -import ( - "path" - "path/filepath" - "testing" - - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/config/testconfig" - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/hugolib/filesystems" - "github.com/gohugoio/hugo/hugolib/paths" - "github.com/gohugoio/hugo/media" - - "github.com/spf13/afero" - - "github.com/evanw/esbuild/pkg/api" - - qt "github.com/frankban/quicktest" -) - -// This test is added to test/warn against breaking the "stability" of the -// cache key. It's sometimes needed to break this, but should be avoided if possible. -func TestOptionKey(t *testing.T) { - c := qt.New(t) - - opts := map[string]any{ - "TargetPath": "foo", - "Target": "es2018", - } - - key := (&buildTransformation{optsm: opts}).Key() - - c.Assert(key.Value(), qt.Equals, "jsbuild_1533819657654811600") -} - -func TestToBuildOptions(t *testing.T) { - c := qt.New(t) - - opts, err := toBuildOptions(Options{mediaType: media.Builtin.JavascriptType}) - - c.Assert(err, qt.IsNil) - c.Assert(opts, qt.DeepEquals, api.BuildOptions{ - Bundle: true, - Target: api.ESNext, - Format: api.FormatIIFE, - Stdin: &api.StdinOptions{ - Loader: api.LoaderJS, - }, - }) - - opts, err = toBuildOptions(Options{ - Target: "es2018", - Format: "cjs", - Minify: true, - mediaType: media.Builtin.JavascriptType, - AvoidTDZ: true, - }) - c.Assert(err, qt.IsNil) - c.Assert(opts, qt.DeepEquals, api.BuildOptions{ - Bundle: true, - Target: api.ES2018, - Format: api.FormatCommonJS, - MinifyIdentifiers: true, - MinifySyntax: true, - MinifyWhitespace: true, - Stdin: &api.StdinOptions{ - Loader: api.LoaderJS, - }, - }) - - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "inline", - }) - c.Assert(err, qt.IsNil) - c.Assert(opts, qt.DeepEquals, api.BuildOptions{ - Bundle: true, - Target: api.ES2018, - Format: api.FormatCommonJS, - MinifyIdentifiers: true, - MinifySyntax: true, - MinifyWhitespace: true, - Sourcemap: api.SourceMapInline, - Stdin: &api.StdinOptions{ - Loader: api.LoaderJS, - }, - }) - - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "inline", - }) - c.Assert(err, qt.IsNil) - c.Assert(opts, qt.DeepEquals, api.BuildOptions{ - Bundle: true, - Target: api.ES2018, - Format: api.FormatCommonJS, - MinifyIdentifiers: true, - MinifySyntax: true, - MinifyWhitespace: true, - Sourcemap: api.SourceMapInline, - Stdin: &api.StdinOptions{ - Loader: api.LoaderJS, - }, - }) - - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "external", - }) - c.Assert(err, qt.IsNil) - c.Assert(opts, qt.DeepEquals, api.BuildOptions{ - Bundle: true, - Target: api.ES2018, - Format: api.FormatCommonJS, - MinifyIdentifiers: true, - MinifySyntax: true, - MinifyWhitespace: true, - Sourcemap: api.SourceMapExternal, - Stdin: &api.StdinOptions{ - Loader: api.LoaderJS, - }, - }) - - opts, err = toBuildOptions(Options{ - mediaType: media.Builtin.JavascriptType, - JSX: "automatic", JSXImportSource: "preact", - }) - c.Assert(err, qt.IsNil) - c.Assert(opts, qt.DeepEquals, api.BuildOptions{ - Bundle: true, - Target: api.ESNext, - Format: api.FormatIIFE, - Stdin: &api.StdinOptions{ - Loader: api.LoaderJS, - }, - JSX: api.JSXAutomatic, - JSXImportSource: "preact", - }) -} - -func TestToBuildOptionsTarget(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - target string - expect api.Target - }{ - {"es2015", api.ES2015}, - {"es2016", api.ES2016}, - {"es2017", api.ES2017}, - {"es2018", api.ES2018}, - {"es2019", api.ES2019}, - {"es2020", api.ES2020}, - {"es2021", api.ES2021}, - {"es2022", api.ES2022}, - {"es2023", api.ES2023}, - {"", api.ESNext}, - {"esnext", api.ESNext}, - } { - c.Run(test.target, func(c *qt.C) { - opts, err := toBuildOptions(Options{ - Target: test.target, - mediaType: media.Builtin.JavascriptType, - }) - c.Assert(err, qt.IsNil) - c.Assert(opts.Target, qt.Equals, test.expect) - }) - } -} - -func TestResolveComponentInAssets(t *testing.T) { - c := qt.New(t) - - for _, test := range []struct { - name string - files []string - impPath string - expect string - }{ - {"Basic, extension", []string{"foo.js", "bar.js"}, "foo.js", "foo.js"}, - {"Basic, no extension", []string{"foo.js", "bar.js"}, "foo", "foo.js"}, - {"Basic, no extension, typescript", []string{"foo.ts", "bar.js"}, "foo", "foo.ts"}, - {"Not found", []string{"foo.js", "bar.js"}, "moo.js", ""}, - {"Not found, double js extension", []string{"foo.js.js", "bar.js"}, "foo.js", ""}, - {"Index file, folder only", []string{"foo/index.js", "bar.js"}, "foo", "foo/index.js"}, - {"Index file, folder and index", []string{"foo/index.js", "bar.js"}, "foo/index", "foo/index.js"}, - {"Index file, folder and index and suffix", []string{"foo/index.js", "bar.js"}, "foo/index.js", "foo/index.js"}, - {"Index ESM file, folder only", []string{"foo/index.esm.js", "bar.js"}, "foo", "foo/index.esm.js"}, - {"Index ESM file, folder and index", []string{"foo/index.esm.js", "bar.js"}, "foo/index", "foo/index.esm.js"}, - {"Index ESM file, folder and index and suffix", []string{"foo/index.esm.js", "bar.js"}, "foo/index.esm.js", "foo/index.esm.js"}, - // We added these index.esm.js cases in v0.101.0. The case below is unlikely to happen in the wild, but add a test - // to document Hugo's behavior. We pick the file with the name index.js; anything else would be breaking. - {"Index and Index ESM file, folder only", []string{"foo/index.esm.js", "foo/index.js", "bar.js"}, "foo", "foo/index.js"}, - - // Issue #8949 - {"Check file before directory", []string{"foo.js", "foo/index.js"}, "foo", "foo.js"}, - } { - c.Run(test.name, func(c *qt.C) { - baseDir := "assets" - mfs := afero.NewMemMapFs() - - for _, filename := range test.files { - c.Assert(afero.WriteFile(mfs, filepath.Join(baseDir, filename), []byte("let foo='bar';"), 0o777), qt.IsNil) - } - - conf := testconfig.GetTestConfig(mfs, config.New()) - fs := hugofs.NewFrom(mfs, conf.BaseConfig()) - - p, err := paths.New(fs, conf) - c.Assert(err, qt.IsNil) - bfs, err := filesystems.NewBase(p, nil) - c.Assert(err, qt.IsNil) - - got := resolveComponentInAssets(bfs.Assets.Fs, test.impPath) - - gotPath := "" - expect := test.expect - if got != nil { - gotPath = filepath.ToSlash(got.Filename) - expect = path.Join(baseDir, test.expect) - } - - c.Assert(gotPath, qt.Equals, expect) - }) - } -} diff --git a/resources/resource_transformers/js/transform.go b/resources/resource_transformers/js/transform.go new file mode 100644 index 000000000..13909e54c --- /dev/null +++ b/resources/resource_transformers/js/transform.go @@ -0,0 +1,68 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package js + +import ( + "io" + "path" + "path/filepath" + + "github.com/gohugoio/hugo/internal/js/esbuild" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/internal" +) + +type buildTransformation struct { + optsm map[string]any + c *Client +} + +func (t *buildTransformation) Key() internal.ResourceTransformationKey { + return internal.NewResourceTransformationKey("jsbuild", t.optsm) +} + +func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { + ctx.OutMediaType = media.Builtin.JavascriptType + + var opts esbuild.Options + + if t.optsm != nil { + optsExt, err := esbuild.DecodeExternalOptions(t.optsm) + if err != nil { + return err + } + opts.ExternalOptions = optsExt + } + + if opts.TargetPath != "" { + ctx.OutPath = opts.TargetPath + } else { + ctx.ReplaceOutPathExtension(".js") + } + + src, err := io.ReadAll(ctx.From) + if err != nil { + return err + } + + opts.SourceDir = filepath.FromSlash(path.Dir(ctx.SourcePath)) + opts.Contents = string(src) + opts.MediaType = ctx.InMediaType + opts.Stdin = true + + _, err = t.c.transform(opts, ctx) + + return err +} diff --git a/resources/resource_transformers/tocss/dartsass/transform.go b/resources/resource_transformers/tocss/dartsass/transform.go index 0d7b57062..77bacc115 100644 --- a/resources/resource_transformers/tocss/dartsass/transform.go +++ b/resources/resource_transformers/tocss/dartsass/transform.go @@ -139,7 +139,7 @@ func (t importResolver) CanonicalizeURL(url string) (string, error) { return url, nil } - filePath, isURL := paths.UrlToFilename(url) + filePath, isURL := paths.UrlStringToFilename(url) var prevDir string var pathDir string if isURL { @@ -195,7 +195,7 @@ func (t importResolver) Load(url string) (godartsass.Import, error) { if url == sass.HugoVarsNamespace { return t.varsStylesheet, nil } - filename, _ := paths.UrlToFilename(url) + filename, _ := paths.UrlStringToFilename(url) b, err := afero.ReadFile(hugofs.Os, filename) sourceSyntax := godartsass.SourceSyntaxSCSS diff --git a/resources/transform.go b/resources/transform.go index 336495e6d..4214067bd 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -52,6 +52,8 @@ var ( _ identity.IdentityGroupProvider = (*resourceAdapterInner)(nil) _ resource.Source = (*resourceAdapter)(nil) _ resource.Identifier = (*resourceAdapter)(nil) + _ targetPathProvider = (*resourceAdapter)(nil) + _ sourcePathProvider = (*resourceAdapter)(nil) _ resource.ResourceNameTitleProvider = (*resourceAdapter)(nil) _ resource.WithResourceMetaProvider = (*resourceAdapter)(nil) _ identity.DependencyManagerProvider = (*resourceAdapter)(nil) @@ -277,6 +279,19 @@ func (r *resourceAdapter) Key() string { return r.target.(resource.Identifier).Key() } +func (r *resourceAdapter) targetPath() string { + r.init(false, false) + return r.target.(targetPathProvider).targetPath() +} + +func (r *resourceAdapter) sourcePath() string { + r.init(false, false) + if sp, ok := r.target.(sourcePathProvider); ok { + return sp.sourcePath() + } + return "" +} + func (r *resourceAdapter) MediaType() media.Type { r.init(false, false) return r.target.MediaType() diff --git a/tpl/debug/debug.go b/tpl/debug/debug.go index ae1acf5eb..fd676f0e2 100644 --- a/tpl/debug/debug.go +++ b/tpl/debug/debug.go @@ -41,7 +41,7 @@ func New(d *deps.Deps) *Namespace { l := d.Log.InfoCommand("timer") - d.BuildEndListeners.Add(func() { + d.BuildEndListeners.Add(func(...any) bool { type data struct { Name string Count int @@ -84,6 +84,8 @@ func New(d *deps.Deps) *Namespace { } ns.timers = make(map[string][]*timer) + + return false }) return ns diff --git a/tpl/fmt/fmt.go b/tpl/fmt/fmt.go index e9c18360a..264cb1435 100644 --- a/tpl/fmt/fmt.go +++ b/tpl/fmt/fmt.go @@ -30,8 +30,9 @@ func New(d *deps.Deps) *Namespace { logger: d.Log, } - d.BuildStartListeners.Add(func() { + d.BuildStartListeners.Add(func(...any) bool { ns.logger.Reset() + return false }) return ns diff --git a/tpl/js/init.go b/tpl/js/init.go index 69d7c7275..97e76c33d 100644 --- a/tpl/js/init.go +++ b/tpl/js/init.go @@ -24,7 +24,10 @@ const name = "js" func init() { f := func(d *deps.Deps) *internal.TemplateFuncsNamespace { - ctx := New(d) + ctx, err := New(d) + if err != nil { + panic(err) + } ns := &internal.TemplateFuncsNamespace{ Name: name, diff --git a/tpl/js/js.go b/tpl/js/js.go index c68e0af92..b686b76a7 100644 --- a/tpl/js/js.go +++ b/tpl/js/js.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,29 +17,47 @@ package js import ( "errors" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/internal/js/esbuild" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/resources/resource_factories/create" "github.com/gohugoio/hugo/resources/resource_transformers/babel" - "github.com/gohugoio/hugo/resources/resource_transformers/js" + jstransform "github.com/gohugoio/hugo/resources/resource_transformers/js" "github.com/gohugoio/hugo/tpl/internal/resourcehelpers" ) // New returns a new instance of the js-namespaced template functions. -func New(deps *deps.Deps) *Namespace { - if deps.ResourceSpec == nil { - return &Namespace{} +func New(d *deps.Deps) (*Namespace, error) { + if d.ResourceSpec == nil { + return &Namespace{}, nil } - return &Namespace{ - client: js.New(deps.BaseFs.Assets, deps.ResourceSpec), - babelClient: babel.New(deps.ResourceSpec), + + batcherClient, err := esbuild.NewBatcherClient(d) + if err != nil { + return nil, err } + + return &Namespace{ + d: d, + jsTransformClient: jstransform.New(d.BaseFs.Assets, d.ResourceSpec), + jsBatcherClient: batcherClient, + jsBatcherStore: maps.NewCache[string, esbuild.Batcher](), + createClient: create.New(d.ResourceSpec), + babelClient: babel.New(d.ResourceSpec), + }, nil } // Namespace provides template functions for the "js" namespace. type Namespace struct { - client *js.Client - babelClient *babel.Client + d *deps.Deps + + jsTransformClient *jstransform.Client + createClient *create.Client + babelClient *babel.Client + jsBatcherClient *esbuild.BatcherClient + jsBatcherStore *maps.Cache[string, esbuild.Batcher] } // Build processes the given Resource with ESBuild. @@ -65,7 +83,24 @@ func (ns *Namespace) Build(args ...any) (resource.Resource, error) { m = map[string]any{"targetPath": targetPath} } - return ns.client.Process(r, m) + return ns.jsTransformClient.Process(r, m) +} + +// Batch creates a new Batcher with the given ID. +// Repeated calls with the same ID will return the same Batcher. +// The ID will be used to name the root directory of the batch. +// Forward slashes in the ID is allowed. +func (ns *Namespace) Batch(id string) (esbuild.Batcher, error) { + if err := esbuild.ValidateBatchID(id, true); err != nil { + return nil, err + } + b, err := ns.jsBatcherStore.GetOrCreate(id, func() (esbuild.Batcher, error) { + return ns.jsBatcherClient.New(id) + }) + if err != nil { + return nil, err + } + return b, nil } // Babel processes the given Resource with Babel. diff --git a/tpl/partials/partials.go b/tpl/partials/partials.go index 8de312fd8..4441b5fa5 100644 --- a/tpl/partials/partials.go +++ b/tpl/partials/partials.go @@ -81,8 +81,9 @@ func New(deps *deps.Deps) *Namespace { cache := &partialCache{cache: lru} deps.BuildStartListeners.Add( - func() { + func(...any) bool { cache.clear() + return false }) return &Namespace{ diff --git a/tpl/tplimpl/embedded/templates/_hugo/build/js/batch-esm-runner.gotmpl b/tpl/tplimpl/embedded/templates/_hugo/build/js/batch-esm-runner.gotmpl new file mode 100644 index 000000000..e1b3b9bc3 --- /dev/null +++ b/tpl/tplimpl/embedded/templates/_hugo/build/js/batch-esm-runner.gotmpl @@ -0,0 +1,16 @@ +{{ range $i, $e := .Scripts -}} + {{ printf "import { %s as Script%d } from %q;" .Export $i .Import }} +{{ end -}} +{{ range $i, $e := .Runners }} + {{ printf "import { %s as Run%d } from %q;" .Export $i .Import }} +{{ end }} +{{/* */}} +let scripts = []; +{{ range $i, $e := .Scripts -}} + scripts.push({{ .RunnerJSON $i }}); +{{ end -}} +{{/* */}} +{{ range $i, $e := .Runners }} + {{ $id := printf "Run%d" $i }} + {{ $id }}(scripts); +{{ end }} diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index fd07db68e..9e2af046d 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -695,13 +695,13 @@ func (t *templateHandler) applyBaseTemplate(overlay, base templateInfo) (tpl.Tem if !base.IsZero() { templ, err = templ.Parse(base.template) if err != nil { - return nil, base.errWithFileContext("parse failed", err) + return nil, base.errWithFileContext("text: base: parse failed", err) } } templ, err = texttemplate.Must(templ.Clone()).Parse(overlay.template) if err != nil { - return nil, overlay.errWithFileContext("parse failed", err) + return nil, overlay.errWithFileContext("text: overlay: parse failed", err) } // The extra lookup is a workaround, see @@ -720,13 +720,13 @@ func (t *templateHandler) applyBaseTemplate(overlay, base templateInfo) (tpl.Tem if !base.IsZero() { templ, err = templ.Parse(base.template) if err != nil { - return nil, base.errWithFileContext("parse failed", err) + return nil, base.errWithFileContext("html: base: parse failed", err) } } templ, err = htmltemplate.Must(templ.Clone()).Parse(overlay.template) if err != nil { - return nil, overlay.errWithFileContext("parse failed", err) + return nil, overlay.errWithFileContext("html: overlay: parse failed", err) } // The extra lookup is a workaround, see diff --git a/tpl/tplimpl/template_funcs.go b/tpl/tplimpl/template_funcs.go index 3daba74a0..d73d4d336 100644 --- a/tpl/tplimpl/template_funcs.go +++ b/tpl/tplimpl/template_funcs.go @@ -251,6 +251,9 @@ func newTemplateExecuter(d *deps.Deps) (texttemplate.Executer, map[string]reflec } func createFuncMap(d *deps.Deps) map[string]any { + if d.TmplFuncMap != nil { + return d.TmplFuncMap + } funcMap := template.FuncMap{} nsMap := make(map[string]any) @@ -292,5 +295,7 @@ func createFuncMap(d *deps.Deps) map[string]any { } } - return funcMap + d.TmplFuncMap = funcMap + + return d.TmplFuncMap } |