diff options
90 files changed, 4686 insertions, 1126 deletions
diff --git a/.gitignore b/.gitignore index 08e830c87..032a643c9 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,7 @@ vendor/*/ *.debug coverage*.out +dock.sh + GoBuilds dist diff --git a/.travis.yml b/.travis.yml index b04528203..f398f2015 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: go sudo: false dist: trusty +env: + HUGO_BUILD_TAGS="extended" git: depth: false go: @@ -18,8 +20,9 @@ install: - go get github.com/magefile/mage - mage -v vendor script: - - mage -v hugoRace + - mage -v test - mage -v check + - mage -v hugo - ./hugo -s docs/ - ./hugo --renderToMemory -s docs/ before_install: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad33414b5..87199a411 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -192,6 +192,12 @@ To list all available commands along with descriptions: mage -l ``` +**Note:** From Hugo 0.43 we have added a build tag, `extended` that adds **SCSS support**. This needs a C compiler installed to build. You can enable this when building by: + +```bash +HUGO_BUILD_TAGS=extended mage install +```` + ### Updating the Hugo Sources If you want to stay in sync with the Hugo repository, you can easily pull down diff --git a/Dockerfile b/Dockerfile index 919b02237..919b02237 100644..100755 --- a/Dockerfile +++ b/Dockerfile diff --git a/Gopkg.lock b/Gopkg.lock index 51fb96c52..8d07f2390 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,6 +3,12 @@ [[projects]] branch = "master" + name = "github.com/BurntSushi/locker" + packages = ["."] + revision = "a6e239ea1c69bff1cfdb20c4b73dadf52f784b6a" + +[[projects]] + branch = "master" name = "github.com/BurntSushi/toml" packages = ["."] revision = "a368813c5e648fee92e5f6c30e3944ff9d5e8895" @@ -69,6 +75,16 @@ revision = "012701e8669671499fc43e9792335a1dcbfe2afb" [[projects]] + branch = "master" + name = "github.com/bep/go-tocss" + packages = [ + "scss", + "scss/libsass", + "tocss" + ] + revision = "2abb118dc8688b6c7df44e12f4152c2bded9b19c" + +[[projects]] name = "github.com/chaseadamsio/goorgeous" packages = ["."] revision = "dcf1ef873b8987bf12596fe6951c48347986eb2f" @@ -109,6 +125,12 @@ [[projects]] branch = "master" + name = "github.com/dsnet/golib" + packages = ["memfile"] + revision = "1ea1667757804fdcccc5a1810e09aba618885ac2" + +[[projects]] + branch = "master" name = "github.com/eknkc/amber" packages = [ ".", @@ -233,6 +255,12 @@ [[projects]] branch = "master" + name = "github.com/mitchellh/hashstructure" + packages = ["."] + revision = "2bca23e0e452137f789efbc8610126fd8b94f73b" + +[[projects]] + branch = "master" name = "github.com/mitchellh/mapstructure" packages = ["."] revision = "00c29f56e2386353d58c599509e8dc3801b0d716" @@ -356,6 +384,42 @@ version = "v1.2.1" [[projects]] + name = "github.com/tdewolff/minify" + packages = [ + ".", + "css", + "html", + "js", + "json", + "svg", + "xml" + ] + revision = "8d72a4127ae33b755e95bffede9b92e396267ce2" + version = "v2.3.5" + +[[projects]] + name = "github.com/tdewolff/parse" + packages = [ + ".", + "buffer", + "css", + "html", + "js", + "json", + "strconv", + "svg", + "xml" + ] + revision = "d739d6fccb0971177e06352fea02d3552625efb1" + version = "v2.3.3" + +[[projects]] + branch = "master" + name = "github.com/wellington/go-libsass" + packages = ["libs"] + revision = "615eaa47ef794d037c1906a0eb7bf85375a5decf" + +[[projects]] name = "github.com/yosssi/ace" packages = ["."] revision = "ea038f4770b6746c3f8f84f14fa60d9fe1205b56" @@ -431,6 +495,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "78b19539f7321429f217fc482de9e7cb4e2edd9b054ba8ec36b1e62bc4281b4f" + inputs-digest = "aaf909f54ae33c5a70f692e19e59834106bcbbe5d16724ff3998907734e32c0b" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index c87b82823..8e6a614f2 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -17,6 +17,14 @@ name = "github.com/bep/gitmap" [[constraint]] + branch = "master" + name = "github.com/bep/go-tocss" + +[[override]] + branch = "master" + name = "github.com/wellington/go-libsass" + +[[constraint]] name = "github.com/chaseadamsio/goorgeous" version = "^1.1.0" @@ -149,3 +157,15 @@ [[constraint]] name = "github.com/bep/debounce" version = "^1.1.0" + +[[constraint]] + name = "github.com/tdewolff/minify" + version = "^2.3.5" + +[[constraint]] + branch = "master" + name = "github.com/BurntSushi/locker" + +[[constraint]] + branch = "master" + name = "github.com/mitchellh/hashstructure" diff --git a/appveyor.yml b/appveyor.yml index d6fbaba1a..3feb4819f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,14 @@ +image: Visual Studio 2015 + init: - - set PATH=%PATH%;C:\MinGW\bin;%GOPATH%\bin + - set PATH=%PATH%;C:\mingw-w64\x86_64-7.3.0-posix-seh-rt_v5-rev0\mingw64\bin;%GOPATH%\bin - go version - go env +environment: + GOPATH: C:\GOPATH\ + HUGO_BUILD_TAGS: extended + # clones and cd's to path clone_folder: C:\GOPATH\src\github.com\gohugoio\hugo diff --git a/commands/commandeer.go b/commands/commandeer.go index 4ca0c4be9..051787f6e 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -16,6 +16,7 @@ package commands import ( "os" "path/filepath" + "regexp" "sync" "time" @@ -46,6 +47,10 @@ type commandeerHugoState struct { type commandeer struct { *commandeerHugoState + // Currently only set when in "fast render mode". But it seems to + // be fast enough that we could maybe just add it for all server modes. + changeDetector *fileChangeDetector + // We need to reuse this on server rebuilds. destinationFs afero.Fs @@ -105,6 +110,68 @@ func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f fla return c, c.loadConfig(mustHaveConfigFile, running) } +type fileChangeDetector struct { + sync.Mutex + current map[string]string + prev map[string]string + + irrelevantRe *regexp.Regexp +} + +func (f *fileChangeDetector) OnFileClose(name, md5sum string) { + f.Lock() + defer f.Unlock() + f.current[name] = md5sum +} + +func (f *fileChangeDetector) changed() []string { + if f == nil { + return nil + } + f.Lock() + defer f.Unlock() + var c []string + for k, v := range f.current { + vv, found := f.prev[k] + if !found || v != vv { + c = append(c, k) + } + } + + return f.filterIrrelevant(c) +} + +func (f *fileChangeDetector) filterIrrelevant(in []string) []string { + var filtered []string + for _, v := range in { + if !f.irrelevantRe.MatchString(v) { + filtered = append(filtered, v) + } + } + return filtered +} + +func (f *fileChangeDetector) PrepareNew() { + if f == nil { + return + } + + f.Lock() + defer f.Unlock() + + if f.current == nil { + f.current = make(map[string]string) + f.prev = make(map[string]string) + return + } + + f.prev = make(map[string]string) + for k, v := range f.current { + f.prev[k] = v + } + f.current = make(map[string]string) +} + func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error { if c.DepsCfg == nil { @@ -202,6 +269,23 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error { fs.Destination = new(afero.MemMapFs) } + doLiveReload := !c.h.buildWatch && !config.GetBool("disableLiveReload") + fastRenderMode := doLiveReload && !config.GetBool("disableFastRender") + + if fastRenderMode { + // For now, fast render mode only. It should, however, be fast enough + // for the full variant, too. + changeDetector := &fileChangeDetector{ + // We use this detector to decide to do a Hot reload of a single path or not. + // We need to filter out source maps and possibly some other to be able + // to make that decision. + irrelevantRe: regexp.MustCompile(`\.map$`), + } + changeDetector.PrepareNew() + fs.Destination = hugofs.NewHashingFs(fs.Destination, changeDetector) + c.changeDetector = changeDetector + } + err = c.initFs(fs) if err != nil { return diff --git a/commands/hugo.go b/commands/hugo.go index 2b847ec95..980189c47 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -474,6 +474,10 @@ func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint6 return numFiles, err } +func (c *commandeer) firstPathSpec() *helpers.PathSpec { + return c.hugo.Sites[0].PathSpec +} + func (c *commandeer) timeTrack(start time.Time, name string) { if c.h.quiet { return @@ -552,8 +556,8 @@ func (c *commandeer) getDirList() ([]string, error) { // SymbolicWalk will log anny ERRORs // Also note that the Dirnames fetched below will contain any relevant theme // directories. - for _, contentDir := range c.hugo.PathSpec.BaseFs.AbsContentDirs { - _ = helpers.SymbolicWalk(c.Fs.Source, contentDir.Value, symLinkWalker) + for _, contentDir := range c.hugo.PathSpec.BaseFs.Content.Dirnames { + _ = helpers.SymbolicWalk(c.Fs.Source, contentDir, symLinkWalker) } for _, staticDir := range c.hugo.PathSpec.BaseFs.Data.Dirnames { @@ -574,6 +578,10 @@ func (c *commandeer) getDirList() ([]string, error) { } } + for _, assetDir := range c.hugo.PathSpec.BaseFs.Assets.Dirnames { + _ = helpers.SymbolicWalk(c.Fs.Source, assetDir, regularWalker) + } + if len(nested) > 0 { for { @@ -818,13 +826,11 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) { // Will block forever trying to write to a channel that nobody is reading if livereload isn't initialized // force refresh when more than one file - if len(staticEvents) > 0 { - for _, ev := range staticEvents { - - path := c.hugo.BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name) - livereload.RefreshPath(path) - } - + if len(staticEvents) == 1 { + ev := staticEvents[0] + path := c.hugo.BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name) + path = c.firstPathSpec().RelURL(helpers.ToSlashTrimLeading(path), false) + livereload.RefreshPath(path) } else { livereload.ForceRefresh() } @@ -832,34 +838,54 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) { } if len(dynamicEvents) > 0 { + partitionedEvents := partitionDynamicEvents( + c.firstPathSpec().BaseFs.SourceFilesystems, + dynamicEvents) + doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") - onePageName := pickOneWriteOrCreatePath(dynamicEvents) + onePageName := pickOneWriteOrCreatePath(partitionedEvents.ContentEvents) c.Logger.FEEDBACK.Println("\nChange detected, rebuilding site") const layout = "2006-01-02 15:04:05.000 -0700" c.Logger.FEEDBACK.Println(time.Now().Format(layout)) + c.changeDetector.PrepareNew() if err := c.rebuildSites(dynamicEvents); err != nil { c.Logger.ERROR.Println("Failed to rebuild site:", err) } if doLiveReload { - navigate := c.Cfg.GetBool("navigateToChanged") - // We have fetched the same page above, but it may have - // changed. - var p *hugolib.Page - - if navigate { - if onePageName != "" { - p = c.hugo.GetContentPage(onePageName) + if len(partitionedEvents.ContentEvents) == 0 && len(partitionedEvents.AssetEvents) > 0 { + changed := c.changeDetector.changed() + if c.changeDetector != nil && len(changed) == 0 { + // Nothing has changed. + continue + } else if len(changed) == 1 { + pathToRefresh := c.firstPathSpec().RelURL(helpers.ToSlashTrimLeading(changed[0]), false) + livereload.RefreshPath(pathToRefresh) + } else { + livereload.ForceRefresh() } - } - if p != nil { - livereload.NavigateToPathForPort(p.RelPermalink(), p.Site.ServerPort()) - } else { - livereload.ForceRefresh() + if len(partitionedEvents.ContentEvents) > 0 { + + navigate := c.Cfg.GetBool("navigateToChanged") + // We have fetched the same page above, but it may have + // changed. + var p *hugolib.Page + + if navigate { + if onePageName != "" { + p = c.hugo.GetContentPage(onePageName) + } + } + + if p != nil { + livereload.NavigateToPathForPort(p.RelPermalink(), p.Site.ServerPort()) + } else { + livereload.ForceRefresh() + } } } } @@ -874,6 +900,26 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) { return watcher, nil } +// dynamicEvents contains events that is considered dynamic, as in "not static". +// Both of these categories will trigger a new build, but the asset events +// does not fit into the "navigate to changed" logic. +type dynamicEvents struct { + ContentEvents []fsnotify.Event + AssetEvents []fsnotify.Event +} + +func partitionDynamicEvents(sourceFs *filesystems.SourceFilesystems, events []fsnotify.Event) (de dynamicEvents) { + for _, e := range events { + if sourceFs.IsAsset(e.Name) { + de.AssetEvents = append(de.AssetEvents, e) + } else { + de.ContentEvents = append(de.ContentEvents, e) + } + } + return + +} + func pickOneWriteOrCreatePath(events []fsnotify.Event) string { name := "" diff --git a/common/errors/errors.go b/common/errors/errors.go new file mode 100644 index 000000000..eff65ff92 --- /dev/null +++ b/common/errors/errors.go @@ -0,0 +1,23 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package errors contains common Hugo errors and error related utilities. +package errors + +import ( + "errors" +) + +// We will, at least to begin with, make some Hugo features (SCSS with libsass) optional, +// and this error is used to signal those situations. +var FeatureNotAvailableErr = errors.New("this feature is not available in your current Hugo version") diff --git a/create/content_template_handler.go b/create/content_template_handler.go index 37eed52cf..02598d4d3 100644 --- a/create/content_template_handler.go +++ b/create/content_template_handler.go @@ -134,7 +134,7 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile return nil, fmt.Errorf("Failed to parse archetype file %q: %s", archetypeFilename, err) } - templ := templateHandler.Lookup(templateName) + templ, _ := templateHandler.Lookup(templateName) var buff bytes.Buffer if err := templ.Execute(&buff, data); err != nil { diff --git a/create/content_test.go b/create/content_test.go index e9d46becf..f3bcc1dd5 100644 --- a/create/content_test.go +++ b/create/content_test.go @@ -88,6 +88,8 @@ func initViper(v *viper.Viper) { v.Set("i18nDir", "i18n") v.Set("theme", "sample") v.Set("archetypeDir", "archetypes") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") } func initFs(fs *hugofs.Fs) error { @@ -191,6 +193,7 @@ func newTestCfg() (*viper.Viper, *hugofs.Fs) { v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") v.Set("archetypeDir", "archetypes") + v.Set("assetDir", "assets") fs := hugofs.NewMem(v) diff --git a/deps/deps.go b/deps/deps.go index d233025d3..b32c7e2e9 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -1,17 +1,18 @@ package deps import ( - "io/ioutil" - "log" - "os" "time" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/metrics" "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/resource" "github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/tpl" jww "github.com/spf13/jwalterweatherman" @@ -30,6 +31,9 @@ type Deps struct { // The templates to use. This will usually implement the full tpl.TemplateHandler. Tmpl tpl.TemplateFinder `json:"-"` + // We use this to parse and execute ad-hoc text templates. + TextTmpl tpl.TemplateParseFinder `json:"-"` + // The file systems to use. Fs *hugofs.Fs `json:"-"` @@ -42,6 +46,9 @@ type Deps struct { // The SourceSpec to use SourceSpec *source.SourceSpec `json:"-"` + // The Resource Spec to use + ResourceSpec *resource.Spec + // The configuration to use Cfg config.Provider `json:"-"` @@ -115,7 +122,7 @@ func New(cfg DepsCfg) (*Deps, error) { } if logger == nil { - logger = jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) + logger = loggers.NewErrorLogger() } if fs == nil { @@ -129,6 +136,11 @@ func New(cfg DepsCfg) (*Deps, error) { return nil, err } + resourceSpec, err := resource.NewSpec(ps, logger, cfg.MediaTypes) + if err != nil { + return nil, err + } + contentSpec, err := helpers.NewContentSpec(cfg.Language) if err != nil { return nil, err @@ -153,6 +165,7 @@ func New(cfg DepsCfg) (*Deps, error) { PathSpec: ps, ContentSpec: contentSpec, SourceSpec: sp, + ResourceSpec: resourceSpec, Cfg: cfg.Language, Language: cfg.Language, Timeout: time.Duration(timeoutms) * time.Millisecond, @@ -167,7 +180,8 @@ func New(cfg DepsCfg) (*Deps, error) { // ForLanguage creates a copy of the Deps with the language dependent // parts switched out. -func (d Deps) ForLanguage(l *langs.Language) (*Deps, error) { +func (d Deps) ForLanguage(cfg DepsCfg) (*Deps, error) { + l := cfg.Language var err error d.PathSpec, err = helpers.NewPathSpecWithBaseBaseFsProvided(d.Fs, l, d.BaseFs) @@ -180,6 +194,11 @@ func (d Deps) ForLanguage(l *langs.Language) (*Deps, error) { return nil, err } + d.ResourceSpec, err = resource.NewSpec(d.PathSpec, d.Log, cfg.MediaTypes) + if err != nil { + return nil, err + } + d.Cfg = l d.Language = l @@ -212,6 +231,9 @@ type DepsCfg struct { // The configuration to use. Cfg config.Provider + // The media types configured. + MediaTypes media.Types + // Template handling. TemplateProvider ResourceProvider WithTemplate func(templ tpl.TemplateHandler) error diff --git a/helpers/general.go b/helpers/general.go index b442b1eb4..ab66376c3 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -356,7 +356,7 @@ func MD5String(f string) string { // MD5FromFileFast creates a MD5 hash from the given file. It only reads parts of // the file for speed, so don't use it if the files are very subtly different. // It will not close the file. -func MD5FromFileFast(f afero.File) (string, error) { +func MD5FromFileFast(r io.ReadSeeker) (string, error) { const ( // Do not change once set in stone! maxChunks = 8 @@ -369,7 +369,7 @@ func MD5FromFileFast(f afero.File) (string, error) { for i := 0; i < maxChunks; i++ { if i > 0 { - _, err := f.Seek(seek, 0) + _, err := r.Seek(seek, 0) if err != nil { if err == io.EOF { break @@ -378,7 +378,7 @@ func MD5FromFileFast(f afero.File) (string, error) { } } - _, err := io.ReadAtLeast(f, buff, peekSize) + _, err := io.ReadAtLeast(r, buff, peekSize) if err != nil { if err == io.EOF || err == io.ErrUnexpectedEOF { h.Write(buff) diff --git a/helpers/path.go b/helpers/path.go index 76f13d653..92ce4079f 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -90,6 +90,11 @@ func (p *PathSpec) MakePathSanitized(s string) string { return strings.ToLower(p.MakePath(s)) } +// ToSlashTrimLeading is just a filepath.ToSlaas with an added / prefix trimmer. +func ToSlashTrimLeading(s string) string { + return strings.TrimPrefix(filepath.ToSlash(s), "/") +} + // MakeTitle converts the path given to a suitable title, trimming whitespace // and replacing hyphens with whitespace. func MakeTitle(inpath string) string { @@ -222,12 +227,22 @@ func GetDottedRelativePath(inPath string) string { return dottedPath } +// ExtNoDelimiter takes a path and returns the extension, excluding the delmiter, i.e. "md". +func ExtNoDelimiter(in string) string { + return strings.TrimPrefix(Ext(in), ".") +} + // Ext takes a path and returns the extension, including the delmiter, i.e. ".md". func Ext(in string) string { _, ext := fileAndExt(in, fpb) return ext } +// PathAndExt is the same as FileAndExt, but it uses the path package. +func PathAndExt(in string) (string, string) { + return fileAndExt(in, pb) +} + // FileAndExt takes a path and returns the file and extension separated, // the extension including the delmiter, i.e. ".md". func FileAndExt(in string) (string, string) { diff --git a/helpers/path_test.go b/helpers/path_test.go index 2c6cb9f37..c249a519d 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -78,6 +78,9 @@ func TestMakePathSanitized(t *testing.T) { v.Set("dataDir", "data") v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") + v.Set("assetDir", "assets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") v.Set("archetypeDir", "archetypes") l := langs.NewDefaultLanguage(v) @@ -475,6 +478,7 @@ func createTempDirWithNonZeroLengthFiles() (string, error) { return "", fileErr } byteString := []byte("byteString") + fileErr = ioutil.WriteFile(f.Name(), byteString, 0644) if fileErr != nil { // delete the file @@ -585,6 +589,11 @@ func TestAbsPathify(t *testing.T) { } +func TestExtNoDelimiter(t *testing.T) { + assert := require.New(t) + assert.Equal("json", ExtNoDelimiter(filepath.FromSlash("/my/data.json"))) +} + func TestFilename(t *testing.T) { type test struct { input, expected string diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index fda1c9ea2..c9da4f129 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -38,6 +38,9 @@ func newTestCfg() *viper.Viper { v.Set("dataDir", "data") v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") + v.Set("assetDir", "assets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") v.Set("archetypeDir", "archetypes") return v } diff --git a/hugofs/basepath_real_filename_fs.go b/hugofs/basepath_real_filename_fs.go new file mode 100644 index 000000000..d0c56df74 --- /dev/null +++ b/hugofs/basepath_real_filename_fs.go @@ -0,0 +1,84 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "os" + + "github.com/spf13/afero" +) + +// RealFilenameInfo is a thin wrapper around os.FileInfo adding the real filename. +type RealFilenameInfo interface { + os.FileInfo + + // This is the real filename to the file in the underlying filesystem. + RealFilename() string +} + +type realFilenameInfo struct { + os.FileInfo + realFilename string +} + +func (f *realFilenameInfo) RealFilename() string { + return f.realFilename +} + +func NewBasePathRealFilenameFs(base *afero.BasePathFs) *BasePathRealFilenameFs { + return &BasePathRealFilenameFs{BasePathFs: base} +} + +// This is a thin wrapper around afero.BasePathFs that provides the real filename +// in Stat and LstatIfPossible. +type BasePathRealFilenameFs struct { + *afero.BasePathFs +} + +func (b *BasePathRealFilenameFs) Stat(name string) (os.FileInfo, error) { + fi, err := b.BasePathFs.Stat(name) + if err != nil { + return nil, err + } + + if _, ok := fi.(RealFilenameInfo); ok { + return fi, nil + } + + filename, err := b.RealPath(name) + if err != nil { + return nil, &os.PathError{Op: "stat", Path: name, Err: err} + } + + return &realFilenameInfo{FileInfo: fi, realFilename: filename}, nil +} + +func (b *BasePathRealFilenameFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + + fi, ok, err := b.BasePathFs.LstatIfPossible(name) + if err != nil { + return nil, false, err + } + + if _, ok := fi.(RealFilenameInfo); ok { + return fi, ok, nil + } + + filename, err := b.RealPath(name) + if err != nil { + return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err} + } + + return &realFilenameInfo{FileInfo: fi, realFilename: filename}, ok, nil +} diff --git a/hugofs/hashing_fs.go b/hugofs/hashing_fs.go new file mode 100644 index 000000000..2de027ce2 --- /dev/null +++ b/hugofs/hashing_fs.go @@ -0,0 +1,96 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "crypto/md5" + "encoding/hex" + "hash" + "os" + + "github.com/spf13/afero" +) + +var ( + _ afero.Fs = (*md5HashingFs)(nil) +) + +// FileHashReceiver will receive the filename an the content's MD5 sum on file close. +type FileHashReceiver interface { + OnFileClose(name, md5sum string) +} + +type md5HashingFs struct { + afero.Fs + hashReceiver FileHashReceiver +} + +// NewHashingFs creates a new filesystem that will receive MD5 checksums of +// any written file content on Close. Note that this is probably not a good +// idea for "full build" situations, but when doing fast render mode, the amount +// of files published is low, and it would be really nice to know exactly which +// of these files where actually changed. +// Note that this will only work for file operations that use the io.Writer +// to write content to file, but that is fine for the "publish content" use case. +func NewHashingFs(delegate afero.Fs, hashReceiver FileHashReceiver) afero.Fs { + return &md5HashingFs{Fs: delegate, hashReceiver: hashReceiver} +} + +func (fs *md5HashingFs) Create(name string) (afero.File, error) { + f, err := fs.Fs.Create(name) + if err == nil { + f = fs.wrapFile(f) + } + return f, err +} + +func (fs *md5HashingFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + f, err := fs.Fs.OpenFile(name, flag, perm) + if err == nil && isWrite(flag) { + f = fs.wrapFile(f) + } + return f, err +} + +func (fs *md5HashingFs) wrapFile(f afero.File) afero.File { + return &hashingFile{File: f, h: md5.New(), hashReceiver: fs.hashReceiver} +} + +func isWrite(flag int) bool { + return flag&os.O_RDWR != 0 || flag&os.O_WRONLY != 0 +} + +func (fs *md5HashingFs) Name() string { + return "md5HashingFs" +} + +type hashingFile struct { + hashReceiver FileHashReceiver + h hash.Hash + afero.File +} + +func (h *hashingFile) Write(p []byte) (n int, err error) { + n, err = h.File.Write(p) + if err != nil { + return + } + return h.h.Write(p) +} + +func (h *hashingFile) Close() error { + sum := hex.EncodeToString(h.h.Sum(nil)) + h.hashReceiver.OnFileClose(h.Name(), sum) + return h.File.Close() +} diff --git a/hugofs/hashing_fs_test.go b/hugofs/hashing_fs_test.go new file mode 100644 index 000000000..b690630ed --- /dev/null +++ b/hugofs/hashing_fs_test.go @@ -0,0 +1,53 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +type testHashReceiver struct { + sum string + name string +} + +func (t *testHashReceiver) OnFileClose(name, md5hash string) { + t.name = name + t.sum = md5hash +} + +func TestHashingFs(t *testing.T) { + assert := require.New(t) + + fs := afero.NewMemMapFs() + observer := &testHashReceiver{} + ofs := NewHashingFs(fs, observer) + + f, err := ofs.Create("hashme") + assert.NoError(err) + _, err = f.Write([]byte("content")) + assert.NoError(err) + assert.NoError(f.Close()) + assert.Equal("9a0364b9e99bb480dd25e1f0284c8555", observer.sum) + assert.Equal("hashme", observer.name) + + f, err = ofs.Create("nowrites") + assert.NoError(err) + assert.NoError(f.Close()) + assert.Equal("d41d8cd98f00b204e9800998ecf8427e", observer.sum) + +} diff --git a/hugolib/alias.go b/hugolib/alias.go index dbb864384..3b053130e 100644 --- a/hugolib/alias.go +++ b/hugolib/alias.go @@ -59,13 +59,14 @@ func (a aliasHandler) renderAlias(isXHTML bool, permalink string, page *Page) (i t = "alias-xhtml" } - var templ *tpl.TemplateAdapter + var templ tpl.Template + var found bool if a.t != nil { - templ = a.t.Lookup("alias.html") + templ, found = a.t.Lookup("alias.html") } - if templ == nil { + if !found { def := defaultAliasTemplates.Lookup(t) if def != nil { templ = &tpl.TemplateAdapter{Template: def} diff --git a/hugolib/alias_test.go b/hugolib/alias_test.go index 04c5b4358..da1b80b70 100644 --- a/hugolib/alias_test.go +++ b/hugolib/alias_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/hugolib/config.go b/hugolib/config.go index dec5b870d..87f97f3a5 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -411,6 +411,7 @@ func loadDefaultSettingsFor(v *viper.Viper) error { v.SetDefault("metaDataFormat", "toml") v.SetDefault("contentDir", "content") v.SetDefault("layoutDir", "layouts") + v.SetDefault("assetDir", "assets") v.SetDefault("staticDir", "static") v.SetDefault("resourceDir", "resources") v.SetDefault("archetypeDir", "archetypes") diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index deecd69a5..d4a7fcde7 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -28,7 +28,6 @@ import ( "fmt" - "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/hugolib/paths" "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" @@ -45,20 +44,10 @@ var filePathSeparator = string(filepath.Separator) // to underline that even if they can be composites, they all have a base path set to a specific // resource folder, e.g "/my-project/content". So, no absolute filenames needed. type BaseFs struct { - // TODO(bep) make this go away - AbsContentDirs []types.KeyValueStr - - // The filesystem used to capture content. This can be a composite and - // language aware file system. - ContentFs afero.Fs // SourceFilesystems contains the different source file systems. *SourceFilesystems - // The filesystem used to store resources (processed images etc.). - // This usually maps to /my-project/resources. - ResourcesFs afero.Fs - // The filesystem used to publish the rendered site. // This usually maps to /my-project/public. PublishFs afero.Fs @@ -71,35 +60,31 @@ type BaseFs struct { // RelContentDir tries to create a path relative to the content root from // the given filename. The return value is the path and language code. -func (b *BaseFs) RelContentDir(filename string) (string, string) { - for _, dir := range b.AbsContentDirs { - if strings.HasPrefix(filename, dir.Value) { - rel := strings.TrimPrefix(filename, dir.Value) - return strings.TrimPrefix(rel, filePathSeparator), dir.Key +func (b *BaseFs) RelContentDir(filename string) string { + for _, dirname := range b.SourceFilesystems.Content.Dirnames { + if strings.HasPrefix(filename, dirname) { + rel := strings.TrimPrefix(filename, dirname) + return strings.TrimPrefix(rel, filePathSeparator) } } // Either not a content dir or already relative. - return filename, "" -} - -// IsContent returns whether the given filename is in the content filesystem. -func (b *BaseFs) IsContent(filename string) bool { - for _, dir := range b.AbsContentDirs { - if strings.HasPrefix(filename, dir.Value) { - return true - } - } - return false + return filename } // SourceFilesystems contains the different source file systems. These can be // composite file systems (theme and project etc.), and they have all root // set to the source type the provides: data, i18n, static, layouts. type SourceFilesystems struct { + Content *SourceFilesystem Data *SourceFilesystem I18n *SourceFilesystem Layouts *SourceFilesystem Archetypes *SourceFilesystem + Assets *SourceFilesystem + Resources *SourceFilesystem + + // This is a unified read-only view of the project's and themes' workdir. + Work *SourceFilesystem // When in multihost we have one static filesystem per language. The sync // static files is currently done outside of the Hugo build (where there is @@ -112,8 +97,14 @@ type SourceFilesystems struct { // i18n, layouts, static) and additional metadata to be able to use that filesystem // in server mode. type SourceFilesystem struct { + // This is a virtual composite filesystem. It expects path relative to a context. Fs afero.Fs + // This is the base source filesystem. In real Hugo, this will be the OS filesystem. + // Use this if you need to resolve items in Dirnames below. + SourceFs afero.Fs + + // Dirnames is absolute filenames to the directories in this filesystem. Dirnames []string // When syncing a source folder to the target (e.g. /public), this may @@ -122,6 +113,50 @@ type SourceFilesystem struct { PublishFolder string } +// ContentStaticAssetFs will create a new composite filesystem from the content, +// static, and asset filesystems. The site language is needed to pick the correct static filesystem. +// The order is content, static and then assets. +// TODO(bep) check usage +func (s SourceFilesystems) ContentStaticAssetFs(lang string) afero.Fs { + staticFs := s.StaticFs(lang) + + base := afero.NewCopyOnWriteFs(s.Assets.Fs, staticFs) + return afero.NewCopyOnWriteFs(base, s.Content.Fs) + +} + +// StaticFs returns the static filesystem for the given language. +// This can be a composite filesystem. +func (s SourceFilesystems) StaticFs(lang string) afero.Fs { + var staticFs afero.Fs = hugofs.NoOpFs + + if fs, ok := s.Static[lang]; ok { + staticFs = fs.Fs + } else if fs, ok := s.Static[""]; ok { + staticFs = fs.Fs + } + + return staticFs +} + +// StatResource looks for a resource in these filesystems in order: static, assets and finally content. +// If found in any of them, it returns FileInfo and the relevant filesystem. +// Any non os.IsNotExist error will be returned. +// An os.IsNotExist error wil be returned only if all filesystems return such an error. +// Note that if we only wanted to find the file, we could create a composite Afero fs, +// but we also need to know which filesystem root it lives in. +func (s SourceFilesystems) StatResource(lang, filename string) (fi os.FileInfo, fs afero.Fs, err error) { + for _, fsToCheck := range []afero.Fs{s.StaticFs(lang), s.Assets.Fs, s.Content.Fs} { + fs = fsToCheck + fi, err = fs.Stat(filename) + if err == nil || !os.IsNotExist(err) { + return + } + } + // Not found. + return +} + // IsStatic returns true if the given filename is a member of one of the static // filesystems. func (s SourceFilesystems) IsStatic(filename string) bool { @@ -133,6 +168,11 @@ func (s SourceFilesystems) IsStatic(filename string) bool { return false } +// IsContent returns true if the given filename is a member of the content filesystem. +func (s SourceFilesystems) IsContent(filename string) bool { + return s.Content.Contains(filename) +} + // IsLayout returns true if the given filename is a member of the layouts filesystem. func (s SourceFilesystems) IsLayout(filename string) bool { return s.Layouts.Contains(filename) @@ -143,6 +183,11 @@ func (s SourceFilesystems) IsData(filename string) bool { return s.Data.Contains(filename) } +// IsAsset returns true if the given filename is a member of the data filesystem. +func (s SourceFilesystems) IsAsset(filename string) bool { + return s.Assets.Contains(filename) +} + // IsI18n returns true if the given filename is a member of the i18n filesystem. func (s SourceFilesystems) IsI18n(filename string) bool { return s.I18n.Contains(filename) @@ -171,6 +216,18 @@ func (d *SourceFilesystem) MakePathRelative(filename string) string { return "" } +func (d *SourceFilesystem) RealFilename(rel string) string { + fi, err := d.Fs.Stat(rel) + if err != nil { + return rel + } + if realfi, ok := fi.(hugofs.RealFilenameInfo); ok { + return realfi.RealFilename() + } + + return rel +} + // Contains returns whether the given filename is a member of the current filesystem. func (d *SourceFilesystem) Contains(filename string) bool { for _, dir := range d.Dirnames { @@ -181,6 +238,20 @@ func (d *SourceFilesystem) Contains(filename string) bool { return false } +// RealDirs gets a list of absolute paths to directorys starting from the given +// path. +func (d *SourceFilesystem) RealDirs(from string) []string { + var dirnames []string + for _, dir := range d.Dirnames { + dirname := filepath.Join(dir, from) + + if _, err := hugofs.Os.Stat(dirname); err == nil { + dirnames = append(dirnames, dirname) + } + } + return dirnames +} + // WithBaseFs allows reuse of some potentially expensive to create parts that remain // the same across sites/languages. func WithBaseFs(b *BaseFs) func(*BaseFs) error { @@ -191,11 +262,15 @@ func WithBaseFs(b *BaseFs) func(*BaseFs) error { } } +func newRealBase(base afero.Fs) afero.Fs { + return hugofs.NewBasePathRealFilenameFs(base.(*afero.BasePathFs)) + +} + // NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { fs := p.Fs - resourcesFs := afero.NewBasePathFs(fs.Source, p.AbsResourcesDir) publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir) contentFs, absContentDirs, err := createContentFs(fs.Source, p.WorkingDir, p.DefaultContentLanguage, p.Languages) @@ -209,17 +284,14 @@ func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { if i == j { continue } - if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) { + if strings.HasPrefix(d1, d2) || strings.HasPrefix(d2, d1) { return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2) } } } b := &BaseFs{ - AbsContentDirs: absContentDirs, - ContentFs: contentFs, - ResourcesFs: resourcesFs, - PublishFs: publishFs, + PublishFs: publishFs, } for _, opt := range options { @@ -234,6 +306,12 @@ func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { return nil, err } + sourceFilesystems.Content = &SourceFilesystem{ + SourceFs: fs.Source, + Fs: contentFs, + Dirnames: absContentDirs, + } + b.SourceFilesystems = sourceFilesystems b.themeFs = builder.themeFs b.AbsThemeDirs = builder.absThemeDirs @@ -281,18 +359,39 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { } b.result.I18n = sfs - sfs, err = b.createFs("layoutDir", "layouts") + sfs, err = b.createFs(false, true, "layoutDir", "layouts") if err != nil { return nil, err } b.result.Layouts = sfs - sfs, err = b.createFs("archetypeDir", "archetypes") + sfs, err = b.createFs(false, true, "archetypeDir", "archetypes") if err != nil { return nil, err } b.result.Archetypes = sfs + sfs, err = b.createFs(false, true, "assetDir", "assets") + if err != nil { + return nil, err + } + b.result.Assets = sfs + + sfs, err = b.createFs(true, false, "resourceDir", "resources") + if err != nil { + return nil, err + } + + b.result.Resources = sfs + + err = b.createStaticFs() + + sfs, err = b.createFs(false, true, "", "") + if err != nil { + return nil, err + } + b.result.Work = sfs + err = b.createStaticFs() if err != nil { return nil, err @@ -301,23 +400,38 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { return b.result, nil } -func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*SourceFilesystem, error) { - s := &SourceFilesystem{} - dir := b.p.Cfg.GetString(dirKey) - if dir == "" { - return s, fmt.Errorf("config %q not set", dirKey) +func (b *sourceFilesystemsBuilder) createFs( + mkdir bool, + readOnly bool, + dirKey, themeFolder string) (*SourceFilesystem, error) { + s := &SourceFilesystem{ + SourceFs: b.p.Fs.Source, + } + var dir string + if dirKey != "" { + dir = b.p.Cfg.GetString(dirKey) + if dir == "" { + return s, fmt.Errorf("config %q not set", dirKey) + } } var fs afero.Fs absDir := b.p.AbsPathify(dir) - if b.existsInSource(absDir) { - fs = afero.NewBasePathFs(b.p.Fs.Source, absDir) + existsInSource := b.existsInSource(absDir) + if !existsInSource && mkdir { + // We really need this directory. Make it. + if err := b.p.Fs.Source.MkdirAll(absDir, 0777); err == nil { + existsInSource = true + } + } + if existsInSource { + fs = newRealBase(afero.NewBasePathFs(b.p.Fs.Source, absDir)) s.Dirnames = []string{absDir} } if b.hasTheme { - themeFolderFs := afero.NewBasePathFs(b.themeFs, themeFolder) + themeFolderFs := newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)) if fs == nil { fs = themeFolderFs } else { @@ -334,8 +448,10 @@ func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*Source if fs == nil { s.Fs = hugofs.NoOpFs - } else { + } else if readOnly { s.Fs = afero.NewReadOnlyFs(fs) + } else { + s.Fs = fs } return s, nil @@ -344,7 +460,9 @@ func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*Source // Used for data, i18n -- we cannot use overlay filsesystems for those, but we need // to keep a strict order. func (b *sourceFilesystemsBuilder) createRootMappingFs(dirKey, themeFolder string) (*SourceFilesystem, error) { - s := &SourceFilesystem{} + s := &SourceFilesystem{ + SourceFs: b.p.Fs.Source, + } projectDir := b.p.Cfg.GetString(dirKey) if projectDir == "" { @@ -396,7 +514,9 @@ func (b *sourceFilesystemsBuilder) createStaticFs() error { if isMultihost { for _, l := range b.p.Languages { - s := &SourceFilesystem{PublishFolder: l.Lang} + s := &SourceFilesystem{ + SourceFs: b.p.Fs.Source, + PublishFolder: l.Lang} staticDirs := removeDuplicatesKeepRight(getStaticDirs(l)) if len(staticDirs) == 0 { continue @@ -424,7 +544,10 @@ func (b *sourceFilesystemsBuilder) createStaticFs() error { return nil } - s := &SourceFilesystem{} + s := &SourceFilesystem{ + SourceFs: b.p.Fs.Source, + } + var staticDirs []string for _, l := range b.p.Languages { @@ -451,7 +574,7 @@ func (b *sourceFilesystemsBuilder) createStaticFs() error { if b.hasTheme { themeFolder := "static" - fs = afero.NewCopyOnWriteFs(afero.NewBasePathFs(b.themeFs, themeFolder), fs) + fs = afero.NewCopyOnWriteFs(newRealBase(afero.NewBasePathFs(b.themeFs, themeFolder)), fs) for _, absThemeDir := range b.absThemeDirs { s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder)) } @@ -484,7 +607,7 @@ func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { func createContentFs(fs afero.Fs, workingDir, defaultContentLanguage string, - languages langs.Languages) (afero.Fs, []types.KeyValueStr, error) { + languages langs.Languages) (afero.Fs, []string, error) { var contentLanguages langs.Languages var contentDirSeen = make(map[string]bool) @@ -511,7 +634,7 @@ func createContentFs(fs afero.Fs, } - var absContentDirs []types.KeyValueStr + var absContentDirs []string fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs) return fs, absContentDirs, err @@ -522,7 +645,7 @@ func createContentOverlayFs(source afero.Fs, workingDir string, languages langs.Languages, languageSet map[string]bool, - absContentDirs *[]types.KeyValueStr) (afero.Fs, error) { + absContentDirs *[]string) (afero.Fs, error) { if len(languages) == 0 { return source, nil } @@ -548,7 +671,7 @@ func createContentOverlayFs(source afero.Fs, return nil, fmt.Errorf("invalid content dir %q: Path is too short", absContentDir) } - *absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir}) + *absContentDirs = append(*absContentDirs, absContentDir) overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir)) if len(languages) == 1 { @@ -597,10 +720,10 @@ func createOverlayFs(source afero.Fs, absPaths []string) (afero.Fs, error) { } if len(absPaths) == 1 { - return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])), nil + return afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0]))), nil } - base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])) + base := afero.NewReadOnlyFs(newRealBase(afero.NewBasePathFs(source, absPaths[0]))) overlay, err := createOverlayFs(source, absPaths[1:]) if err != nil { return nil, err diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go index ea09cd8fd..3e043966f 100644 --- a/hugolib/filesystems/basefs_test.go +++ b/hugolib/filesystems/basefs_test.go @@ -60,6 +60,10 @@ theme = ["atheme"] setConfigAndWriteSomeFilesTo(fs.Source, v, "staticDir", "mystatic", 6) setConfigAndWriteSomeFilesTo(fs.Source, v, "dataDir", "mydata", 7) setConfigAndWriteSomeFilesTo(fs.Source, v, "archetypeDir", "myarchetypes", 8) + setConfigAndWriteSomeFilesTo(fs.Source, v, "assetDir", "myassets", 9) + setConfigAndWriteSomeFilesTo(fs.Source, v, "resourceDir", "myrsesource", 10) + + v.Set("publishDir", "public") p, err := paths.New(fs, v) assert.NoError(err) @@ -88,12 +92,15 @@ theme = ["atheme"] _, err = ff.Readdirnames(-1) assert.NoError(err) - checkFileCount(bfs.ContentFs, "", assert, 3) + checkFileCount(bfs.Content.Fs, "", assert, 3) checkFileCount(bfs.I18n.Fs, "", assert, 6) // 4 + 2 themes checkFileCount(bfs.Layouts.Fs, "", assert, 5) checkFileCount(bfs.Static[""].Fs, "", assert, 6) checkFileCount(bfs.Data.Fs, "", assert, 9) // 7 + 2 themes checkFileCount(bfs.Archetypes.Fs, "", assert, 8) + checkFileCount(bfs.Assets.Fs, "", assert, 9) + checkFileCount(bfs.Resources.Fs, "", assert, 10) + checkFileCount(bfs.Work.Fs, "", assert, 57) assert.Equal([]string{filepath.FromSlash("/my/work/mydata"), filepath.FromSlash("/my/work/themes/btheme/data"), filepath.FromSlash("/my/work/themes/atheme/data")}, bfs.Data.Dirnames) @@ -101,15 +108,16 @@ theme = ["atheme"] assert.True(bfs.IsI18n(filepath.Join(workingDir, "myi18n", "file1.txt"))) assert.True(bfs.IsLayout(filepath.Join(workingDir, "mylayouts", "file1.txt"))) assert.True(bfs.IsStatic(filepath.Join(workingDir, "mystatic", "file1.txt"))) + assert.True(bfs.IsAsset(filepath.Join(workingDir, "myassets", "file1.txt"))) + contentFilename := filepath.Join(workingDir, "mycontent", "file1.txt") assert.True(bfs.IsContent(contentFilename)) - rel, _ := bfs.RelContentDir(contentFilename) + rel := bfs.RelContentDir(contentFilename) assert.Equal("file1.txt", rel) } -func TestNewBaseFsEmpty(t *testing.T) { - assert := require.New(t) +func createConfig() *viper.Viper { v := viper.New() v.Set("contentDir", "mycontent") v.Set("i18nDir", "myi18n") @@ -117,18 +125,90 @@ func TestNewBaseFsEmpty(t *testing.T) { v.Set("dataDir", "mydata") v.Set("layoutDir", "mylayouts") v.Set("archetypeDir", "myarchetypes") + v.Set("assetDir", "myassets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") + + return v +} +func TestNewBaseFsEmpty(t *testing.T) { + assert := require.New(t) + v := createConfig() fs := hugofs.NewMem(v) p, err := paths.New(fs, v) + assert.NoError(err) bfs, err := NewBase(p) assert.NoError(err) assert.NotNil(bfs) assert.Equal(hugofs.NoOpFs, bfs.Archetypes.Fs) assert.Equal(hugofs.NoOpFs, bfs.Layouts.Fs) assert.Equal(hugofs.NoOpFs, bfs.Data.Fs) + assert.Equal(hugofs.NoOpFs, bfs.Assets.Fs) assert.Equal(hugofs.NoOpFs, bfs.I18n.Fs) - assert.NotNil(hugofs.NoOpFs, bfs.ContentFs) - assert.NotNil(hugofs.NoOpFs, bfs.Static) + assert.NotNil(bfs.Work.Fs) + assert.NotNil(bfs.Content.Fs) + assert.NotNil(bfs.Static) +} + +func TestRealDirs(t *testing.T) { + assert := require.New(t) + v := createConfig() + fs := hugofs.NewDefault(v) + sfs := fs.Source + + root, err := afero.TempDir(sfs, "", "realdir") + assert.NoError(err) + themesDir, err := afero.TempDir(sfs, "", "themesDir") + assert.NoError(err) + defer func() { + os.RemoveAll(root) + os.RemoveAll(themesDir) + }() + + v.Set("workingDir", root) + v.Set("contentDir", "content") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") + v.Set("themesDir", themesDir) + v.Set("theme", "mytheme") + + assert.NoError(sfs.MkdirAll(filepath.Join(root, "myassets", "scss", "sf1"), 0755)) + assert.NoError(sfs.MkdirAll(filepath.Join(root, "myassets", "scss", "sf2"), 0755)) + assert.NoError(sfs.MkdirAll(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf2"), 0755)) + assert.NoError(sfs.MkdirAll(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf3"), 0755)) + assert.NoError(sfs.MkdirAll(filepath.Join(root, "resources"), 0755)) + assert.NoError(sfs.MkdirAll(filepath.Join(themesDir, "mytheme", "resources"), 0755)) + + assert.NoError(sfs.MkdirAll(filepath.Join(root, "myassets", "js", "f2"), 0755)) + + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "scss", "sf1", "a1.scss")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "scss", "sf2", "a3.scss")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "scss", "a2.scss")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf2", "a3.scss")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(themesDir, "mytheme", "assets", "scss", "sf3", "a4.scss")), []byte("content"), 0755) + + afero.WriteFile(sfs, filepath.Join(filepath.Join(themesDir, "mytheme", "resources", "t1.txt")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "resources", "p1.txt")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "resources", "p2.txt")), []byte("content"), 0755) + + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "js", "f2", "a1.js")), []byte("content"), 0755) + afero.WriteFile(sfs, filepath.Join(filepath.Join(root, "myassets", "js", "a2.js")), []byte("content"), 0755) + + p, err := paths.New(fs, v) + assert.NoError(err) + bfs, err := NewBase(p) + assert.NoError(err) + assert.NotNil(bfs) + checkFileCount(bfs.Assets.Fs, "", assert, 6) + + realDirs := bfs.Assets.RealDirs("scss") + assert.Equal(2, len(realDirs)) + assert.Equal(filepath.Join(root, "myassets/scss"), realDirs[0]) + assert.Equal(filepath.Join(themesDir, "mytheme/assets/scss"), realDirs[len(realDirs)-1]) + + checkFileCount(bfs.Resources.Fs, "", assert, 3) + } func checkFileCount(fs afero.Fs, dirname string, assert *require.Assertions, expected int) { diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index a0ac72d67..8cb3cf2fd 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -21,8 +21,6 @@ import ( "strings" "sync" - "github.com/gohugoio/hugo/resource" - "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/langs" @@ -182,8 +180,10 @@ func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error { continue } + cfg.Language = s.Language + cfg.MediaTypes = s.mediaTypesConfig + if d == nil { - cfg.Language = s.Language cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate) var err error @@ -200,7 +200,7 @@ func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error { } } else { - d, err = d.ForLanguage(s.Language) + d, err = d.ForLanguage(cfg) if err != nil { return err } @@ -208,11 +208,6 @@ func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error { s.Deps = d } - s.resourceSpec, err = resource.NewSpec(s.Deps.PathSpec, s.mediaTypesConfig) - if err != nil { - return err - } - } return nil @@ -701,7 +696,7 @@ func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bu defer m.mu.RUnlock() // Bundles share resources, so we need to start from the virtual root. - relPath, _ := m.pathSpec.RelContentDir(filename) + relPath := m.pathSpec.RelContentDir(filename) dir, name := filepath.Split(relPath) if !strings.HasSuffix(dir, helpers.FilePathSeparator) { dir += helpers.FilePathSeparator diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index cf7c514f6..4c32fa2f6 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -461,7 +461,7 @@ func TestMultiSitesRebuild(t *testing.T) { b.AssertFileContent("public/fr/sect/doc1/index.html", "Single", "Shortcode: Bonjour") b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Single", "Shortcode: Hello") - contentFs := b.H.BaseFs.ContentFs + contentFs := b.H.BaseFs.Content.Fs for i, this := range []struct { preFunc func(t *testing.T) @@ -698,7 +698,7 @@ title = "Svenska" // Regular pages have no children require.Len(t, svPage.Pages, 0) - require.Len(t, svPage.Data["Pages"], 0) + require.Len(t, svPage.data["Pages"], 0) } diff --git a/hugolib/page.go b/hugolib/page.go index 13907c39e..d9a3fe31c 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -21,6 +21,8 @@ import ( "reflect" "unicode" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/langs" @@ -228,7 +230,7 @@ type Page struct { title string Description string Keywords []string - Data map[string]interface{} + data map[string]interface{} pagemeta.PageDates @@ -239,7 +241,8 @@ type Page struct { permalink string relPermalink string - // relative target path without extension and any base path element from the baseURL. + // relative target path without extension and any base path element + // from the baseURL or the language code. // This is used to construct paths in the page resources. relTargetPathBase string // Is set to a forward slashed path if this is a Page resources living in a folder below its owner. @@ -272,12 +275,16 @@ type Page struct { targetPathDescriptorPrototype *targetPathDescriptor } -func stackTrace() string { - trace := make([]byte, 2000) +func stackTrace(lenght int) string { + trace := make([]byte, lenght) runtime.Stack(trace, true) return string(trace) } +func (p *Page) Data() interface{} { + return p.data +} + func (p *Page) initContent() { p.contentInit.Do(func() { @@ -492,6 +499,10 @@ func (p *Page) BundleType() string { return "" } +func (p *Page) MediaType() media.Type { + return media.OctetType +} + type Source struct { Frontmatter []byte Content []byte @@ -1900,7 +1911,7 @@ func (p *Page) prepareLayouts() error { func (p *Page) prepareData(s *Site) error { if p.Kind != KindSection { var pages Pages - p.Data = make(map[string]interface{}) + p.data = make(map[string]interface{}) switch p.Kind { case KindPage: @@ -1919,21 +1930,21 @@ func (p *Page) prepareData(s *Site) error { singular := s.taxonomiesPluralSingular[plural] taxonomy := s.Taxonomies[plural].Get(term) - p.Data[singular] = taxonomy - p.Data["Singular"] = singular - p.Data["Plural"] = plural - p.Data["Term"] = term + p.data[singular] = taxonomy + p.data["Singular"] = singular + p.data["Plural"] = plural + p.data["Term"] = term pages = taxonomy.Pages() case KindTaxonomyTerm: plural := p.sections[0] singular := s.taxonomiesPluralSingular[plural] - p.Data["Singular"] = singular - p.Data["Plural"] = plural - p.Data["Terms"] = s.Taxonomies[plural] + p.data["Singular"] = singular + p.data["Plural"] = plural + p.data["Terms"] = s.Taxonomies[plural] // keep the following just for legacy reasons - p.Data["OrderedIndex"] = p.Data["Terms"] - p.Data["Index"] = p.Data["Terms"] + p.data["OrderedIndex"] = p.data["Terms"] + p.data["Index"] = p.data["Terms"] // A list of all KindTaxonomy pages with matching plural for _, p := range s.findPagesByKind(KindTaxonomy) { @@ -1943,7 +1954,7 @@ func (p *Page) prepareData(s *Site) error { } } - p.Data["Pages"] = pages + p.data["Pages"] = pages p.Pages = pages } diff --git a/hugolib/page_bundler.go b/hugolib/page_bundler.go index e55e0a92b..9ebfe1b88 100644 --- a/hugolib/page_bundler.go +++ b/hugolib/page_bundler.go @@ -144,7 +144,7 @@ func (s *siteContentProcessor) process(ctx context.Context) error { return nil } for _, file := range files { - f, err := s.site.BaseFs.ContentFs.Open(file.Filename()) + f, err := s.site.BaseFs.Content.Fs.Open(file.Filename()) if err != nil { return fmt.Errorf("failed to open assets file: %s", err) } diff --git a/hugolib/page_bundler_capture_test.go b/hugolib/page_bundler_capture_test.go index 14d8a4368..96d113bf7 100644 --- a/hugolib/page_bundler_capture_test.go +++ b/hugolib/page_bundler_capture_test.go @@ -91,7 +91,7 @@ func TestPageBundlerCaptureSymlinks(t *testing.T) { assert := require.New(t) ps, workDir := newTestBundleSymbolicSources(t) - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) + sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) fileStore := &storeFilenames{} logger := loggers.NewErrorLogger() @@ -137,7 +137,7 @@ func TestPageBundlerCaptureBasic(t *testing.T) { ps, err := helpers.NewPathSpec(fs, cfg) assert.NoError(err) - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) + sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) fileStore := &storeFilenames{} @@ -183,7 +183,7 @@ func TestPageBundlerCaptureMultilingual(t *testing.T) { ps, err := helpers.NewPathSpec(fs, cfg) assert.NoError(err) - sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) + sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.Content.Fs) fileStore := &storeFilenames{} c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) diff --git a/hugolib/page_bundler_handlers.go b/hugolib/page_bundler_handlers.go index eca324294..e0eac3ac4 100644 --- a/hugolib/page_bundler_handlers.go +++ b/hugolib/page_bundler_handlers.go @@ -326,9 +326,14 @@ func (c *contentHandlers) createResource() contentHandler { return notHandled } - resource, err := c.s.resourceSpec.NewResourceFromFilename( - ctx.parentPage.subResourceTargetPathFactory, - ctx.source.Filename(), ctx.target) + resource, err := c.s.ResourceSpec.New( + resource.ResourceSourceDescriptor{ + TargetPathBuilder: ctx.parentPage.subResourceTargetPathFactory, + SourceFile: ctx.source, + RelTargetFilename: ctx.target, + URLBase: c.s.GetURLLanguageBasePath(), + TargetPathBase: c.s.GetTargetLanguageBasePath(), + }) return handlerResult{err: err, handled: true, resource: resource} } @@ -336,7 +341,7 @@ func (c *contentHandlers) createResource() contentHandler { func (c *contentHandlers) copyFile() contentHandler { return func(ctx *handlerContext) handlerResult { - f, err := c.s.BaseFs.ContentFs.Open(ctx.source.Filename()) + f, err := c.s.BaseFs.Content.Fs.Open(ctx.source.Filename()) if err != nil { err := fmt.Errorf("failed to open file in copyFile: %s", err) return handlerResult{err: err} diff --git a/hugolib/page_bundler_test.go b/hugolib/page_bundler_test.go index 3af553ec3..811dbf56f 100644 --- a/hugolib/page_bundler_test.go +++ b/hugolib/page_bundler_test.go @@ -37,7 +37,6 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/resource" "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -158,7 +157,6 @@ func TestPageBundlerSiteRegular(t *testing.T) { altFormat := leafBundle1.OutputFormats().Get("CUSTOMO") assert.NotNil(altFormat) - assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/c/logo.png"), image.(resource.Source).AbsSourceFilename()) assert.Equal("https://example.com/2017/pageslug/c/logo.png", image.Permalink()) th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") diff --git a/hugolib/page_collections.go b/hugolib/page_collections.go index 74f7d608c..8395502f5 100644 --- a/hugolib/page_collections.go +++ b/hugolib/page_collections.go @@ -220,6 +220,6 @@ func (c *PageCollections) clearResourceCacheForPage(page *Page) { dir := path.Dir(first.RelPermalink()) dir = strings.TrimPrefix(dir, page.LanguagePrefix()) // This is done to keep the memory usage in check when doing live reloads. - page.s.resourceSpec.DeleteCacheByPrefix(dir) + page.s.ResourceSpec.DeleteCacheByPrefix(dir) } } diff --git a/hugolib/page_output.go b/hugolib/page_output.go index c1550ccd1..6fffbae86 100644 --- a/hugolib/page_output.go +++ b/hugolib/page_output.go @@ -20,6 +20,10 @@ import ( "strings" "sync" + bp "github.com/gohugoio/hugo/bufferpool" + + "github.com/gohugoio/hugo/tpl" + "github.com/gohugoio/hugo/resource" "github.com/gohugoio/hugo/media" @@ -119,15 +123,15 @@ func (p *PageOutput) Render(layout ...string) template.HTML { } for _, layout := range l { - templ := p.s.Tmpl.Lookup(layout) - if templ == nil { + templ, found := p.s.Tmpl.Lookup(layout) + if !found { // This is legacy from when we had only one output format and // HTML templates only. Some have references to layouts without suffix. // We default to good old HTML. - templ = p.s.Tmpl.Lookup(layout + ".html") + templ, found = p.s.Tmpl.Lookup(layout + ".html") } if templ != nil { - res, err := templ.ExecuteToString(p) + res, err := executeToString(templ, p) if err != nil { p.s.DistinctErrorLog.Printf("in .Render: Failed to execute template %q: %s", layout, err) return template.HTML("") @@ -140,6 +144,16 @@ func (p *PageOutput) Render(layout ...string) template.HTML { } +func executeToString(templ tpl.Template, data interface{}) (string, error) { + b := bp.GetBuffer() + defer bp.PutBuffer(b) + if err := templ.Execute(b, data); err != nil { + return "", err + } + return b.String(), nil + +} + func (p *Page) Render(layout ...string) template.HTML { if p.mainPageOutput == nil { panic(fmt.Sprintf("programming error: no mainPageOutput for %q", p.Path())) @@ -265,7 +279,7 @@ func (p *PageOutput) renderResources() error { // mode when the same resource is member of different page bundles. p.deleteResource(i) } else { - p.s.Log.ERROR.Printf("Failed to publish %q for page %q: %s", src.AbsSourceFilename(), p.pathOrTitle(), err) + p.s.Log.ERROR.Printf("Failed to publish Resource for page %q: %s", p.pathOrTitle(), err) } } else { p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Files) diff --git a/hugolib/page_paths.go b/hugolib/page_paths.go index 4d64f4c14..1b2d00ad5 100644 --- a/hugolib/page_paths.go +++ b/hugolib/page_paths.go @@ -139,7 +139,11 @@ func (p *Page) initURLs() error { return err } - p.relTargetPathBase = strings.TrimSuffix(target, f.MediaType.FullSuffix()) + p.relTargetPathBase = strings.TrimPrefix(strings.TrimSuffix(target, f.MediaType.FullSuffix()), "/") + if prefix := p.s.GetLanguagePrefix(); prefix != "" { + // Any language code in the path will be added later. + p.relTargetPathBase = strings.TrimPrefix(p.relTargetPathBase, prefix+"/") + } p.relPermalink = p.s.PathSpec.PrependBasePath(rel) p.layoutDescriptor = p.createLayoutDescriptor() return nil diff --git a/hugolib/page_paths_test.go b/hugolib/page_paths_test.go index 149505ee4..3ca500f17 100644 --- a/hugolib/page_paths_test.go +++ b/hugolib/page_paths_test.go @@ -27,7 +27,7 @@ import ( func TestPageTargetPath(t *testing.T) { - pathSpec := newTestDefaultPathSpec() + pathSpec := newTestDefaultPathSpec(t) noExtNoDelimMediaType := media.TextType noExtNoDelimMediaType.Suffix = "" diff --git a/hugolib/pagination.go b/hugolib/pagination.go index 84ad74b07..58cec576b 100644 --- a/hugolib/pagination.go +++ b/hugolib/pagination.go @@ -289,7 +289,7 @@ func (p *PageOutput) Paginator(options ...interface{}) (*Pager, error) { if p.s.owner.IsMultihost() { pathDescriptor.LangPrefix = "" } - pagers, err := paginatePages(pathDescriptor, p.Data["Pages"], pagerSize) + pagers, err := paginatePages(pathDescriptor, p.data["Pages"], pagerSize) if err != nil { initError = err diff --git a/hugolib/pagination_test.go b/hugolib/pagination_test.go index 61668c3df..94f7301bb 100644 --- a/hugolib/pagination_test.go +++ b/hugolib/pagination_test.go @@ -281,7 +281,7 @@ func doTestPaginator(t *testing.T, useViper bool) { pages := createTestPages(s, 12) n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat) - n1.Data["Pages"] = pages + n1.data["Pages"] = pages var paginator1 *Pager @@ -301,7 +301,7 @@ func doTestPaginator(t *testing.T, useViper bool) { require.Nil(t, err) require.Equal(t, paginator2, paginator1.Next()) - n1.Data["Pages"] = createTestPages(s, 1) + n1.data["Pages"] = createTestPages(s, 1) samePaginator, _ := n1.Paginator() require.Equal(t, paginator1, samePaginator) diff --git a/hugolib/paths/baseURL.go b/hugolib/paths/baseURL.go index 9cb5627ba..de36c8636 100644 --- a/hugolib/paths/baseURL.go +++ b/hugolib/paths/baseURL.go @@ -27,13 +27,21 @@ type BaseURL struct { } func (b BaseURL) String() string { - return b.urlStr + if b.urlStr != "" { + return b.urlStr + } + return b.url.String() } func (b BaseURL) Path() string { return b.url.Path } +// HostURL returns the URL to the host root without any path elements. +func (b BaseURL) HostURL() string { + return strings.TrimSuffix(b.String(), b.Path()) +} + // WithProtocol returns the BaseURL prefixed with the given protocol. // The Protocol is normally of the form "scheme://", i.e. "webcal://". func (b BaseURL) WithProtocol(protocol string) (string, error) { diff --git a/hugolib/paths/baseURL_test.go b/hugolib/paths/baseURL_test.go index af1d2e38d..382a18314 100644 --- a/hugolib/paths/baseURL_test.go +++ b/hugolib/paths/baseURL_test.go @@ -58,4 +58,9 @@ func TestBaseURL(t *testing.T) { require.NoError(t, err) require.Equal(t, "", b.String()) + // BaseURL with sub path + b, err = newBaseURLFromString("http://example.com/sub") + require.NoError(t, err) + require.Equal(t, "http://example.com/sub", b.String()) + require.Equal(t, "http://example.com", b.HostURL()) } diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index cf8792e5a..3be034fef 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -39,11 +39,14 @@ type Paths struct { // Directories // TODO(bep) when we have trimmed down mos of the dirs usage outside of this package, make // these into an interface. - ContentDir string - ThemesDir string - WorkingDir string + ContentDir string + ThemesDir string + WorkingDir string + + // Directories to store Resource related artifacts. AbsResourcesDir string - AbsPublishDir string + + AbsPublishDir string // pagination path handling PaginatePath string @@ -79,12 +82,21 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { return nil, fmt.Errorf("Failed to create baseURL from %q: %s", baseURLstr, err) } - // TODO(bep) contentDir := cfg.GetString("contentDir") workingDir := cfg.GetString("workingDir") resourceDir := cfg.GetString("resourceDir") publishDir := cfg.GetString("publishDir") + if contentDir == "" { + return nil, fmt.Errorf("contentDir not set") + } + if resourceDir == "" { + return nil, fmt.Errorf("resourceDir not set") + } + if publishDir == "" { + return nil, fmt.Errorf("publishDir not set") + } + defaultContentLanguage := cfg.GetString("defaultContentLanguage") var ( @@ -183,6 +195,21 @@ func (p *Paths) Themes() []string { return p.themes } +func (p *Paths) GetTargetLanguageBasePath() string { + if p.Languages.IsMultihost() { + // In a multihost configuration all assets will be published below the language code. + return p.Lang() + } + return p.GetLanguagePrefix() +} + +func (p *Paths) GetURLLanguageBasePath() string { + if p.Languages.IsMultihost() { + return "" + } + return p.GetLanguagePrefix() +} + func (p *Paths) GetLanguagePrefix() string { if !p.multilingual { return "" diff --git a/hugolib/paths/paths_test.go b/hugolib/paths/paths_test.go index 6cadc747f..3bd445b8b 100644 --- a/hugolib/paths/paths_test.go +++ b/hugolib/paths/paths_test.go @@ -30,6 +30,10 @@ func TestNewPaths(t *testing.T) { v.Set("defaultContentLanguageInSubdir", true) v.Set("defaultContentLanguage", "no") v.Set("multilingual", true) + v.Set("contentDir", "content") + v.Set("workingDir", "work") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") p, err := New(fs, v) assert.NoError(err) diff --git a/hugolib/prune_resources.go b/hugolib/prune_resources.go index e9d2bf96e..28802c6f2 100644 --- a/hugolib/prune_resources.go +++ b/hugolib/prune_resources.go @@ -19,23 +19,29 @@ import ( "os" "strings" + "github.com/gohugoio/hugo/helpers" + "github.com/spf13/afero" ) // GC requires a build first. func (h *HugoSites) GC() (int, error) { s := h.Sites[0] - fs := h.PathSpec.BaseFs.ResourcesFs + fs := h.PathSpec.BaseFs.Resources.Fs - imageCacheDir := s.resourceSpec.GenImagePath + imageCacheDir := s.ResourceSpec.GenImagePath if len(imageCacheDir) < 10 { panic("invalid image cache") } + assetsCacheDir := s.ResourceSpec.GenAssetsPath + if len(assetsCacheDir) < 10 { + panic("invalid assets cache") + } - isInUse := func(filename string) bool { + isImageInUse := func(filename string) bool { key := strings.TrimPrefix(filename, imageCacheDir) for _, site := range h.Sites { - if site.resourceSpec.IsInCache(key) { + if site.ResourceSpec.IsInImageCache(key) { return true } } @@ -43,44 +49,68 @@ func (h *HugoSites) GC() (int, error) { return false } - counter := 0 - - err := afero.Walk(fs, imageCacheDir, func(path string, info os.FileInfo, err error) error { - if info == nil { - return nil + isAssetInUse := func(filename string) bool { + key := strings.TrimPrefix(filename, assetsCacheDir) + // These assets are stored in tuplets with an added extension to the key. + key = strings.TrimSuffix(key, helpers.Ext(key)) + for _, site := range h.Sites { + if site.ResourceSpec.ResourceCache.Contains(key) { + return true + } } - if !strings.HasPrefix(path, imageCacheDir) { - return fmt.Errorf("Invalid state, walk outside of resource dir: %q", path) - } + return false + } - if info.IsDir() { - f, err := fs.Open(path) - if err != nil { + walker := func(dirname string, inUse func(filename string) bool) (int, error) { + counter := 0 + err := afero.Walk(fs, dirname, func(path string, info os.FileInfo, err error) error { + if info == nil { return nil } - defer f.Close() - _, err = f.Readdirnames(1) - if err == io.EOF { - // Empty dir. - s.Fs.Source.Remove(path) + + if !strings.HasPrefix(path, dirname) { + return fmt.Errorf("Invalid state, walk outside of resource dir: %q", path) } - return nil - } + if info.IsDir() { + f, err := fs.Open(path) + if err != nil { + return nil + } + defer f.Close() + _, err = f.Readdirnames(1) + if err == io.EOF { + // Empty dir. + s.Fs.Source.Remove(path) + } - inUse := isInUse(path) - if !inUse { - err := fs.Remove(path) - if err != nil && !os.IsNotExist(err) { - s.Log.ERROR.Printf("Failed to remove %q: %s", path, err) - } else { - counter++ + return nil } - } - return nil - }) - return counter, err + inUse := inUse(path) + if !inUse { + err := fs.Remove(path) + if err != nil && !os.IsNotExist(err) { + s.Log.ERROR.Printf("Failed to remove %q: %s", path, err) + } else { + counter++ + } + } + return nil + }) + + return counter, err + } + + imageCounter, err1 := walker(imageCacheDir, isImageInUse) + assetsCounter, err2 := walker(assetsCacheDir, isAssetInUse) + totalCount := imageCounter + assetsCounter + + if err1 != nil { + return totalCount, err1 + } + + return totalCount, err2 } diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go new file mode 100644 index 000000000..b836ef0b7 --- /dev/null +++ b/hugolib/resource_chain_test.go @@ -0,0 +1,210 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "path/filepath" + "testing" + + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/resource/tocss/scss" +) + +func TestResourceChain(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + shouldRun func() bool + prepare func(b *sitesBuilder) + verify func(b *sitesBuilder) + }{ + {"tocss", func() bool { return scss.Supports() }, func(b *sitesBuilder) { + b.WithTemplates("home.html", ` +{{ $scss := resources.Get "scss/styles2.scss" | toCSS }} +{{ $sass := resources.Get "sass/styles3.sass" | toCSS }} +{{ $scssCustomTarget := resources.Get "scss/styles2.scss" | toCSS (dict "targetPath" "styles/main.css") }} +{{ $scssCustomTargetString := resources.Get "scss/styles2.scss" | toCSS "styles/main.css" }} +{{ $scssMin := resources.Get "scss/styles2.scss" | toCSS | minify }} +T1: Len Content: {{ len $scss.Content }}|RelPermalink: {{ $scss.RelPermalink }}|Permalink: {{ $scss.Permalink }}|MediaType: {{ $scss.MediaType.Type }} +T2: Content: {{ $scssMin.Content }}|RelPermalink: {{ $scssMin.RelPermalink }} +T3: Content: {{ len $scssCustomTarget.Content }}|RelPermalink: {{ $scssCustomTarget.RelPermalink }}|MediaType: {{ $scssCustomTarget.MediaType.Type }} +T4: Content: {{ len $scssCustomTargetString.Content }}|RelPermalink: {{ $scssCustomTargetString.RelPermalink }}|MediaType: {{ $scssCustomTargetString.MediaType.Type }} +T5: Content: {{ $sass.Content }}|T5 RelPermalink: {{ $sass.RelPermalink }}| +`) + }, func(b *sitesBuilder) { + b.AssertFileContent("public/index.html", `T1: Len Content: 24|RelPermalink: /scss/styles2.css|Permalink: http://example.com/scss/styles2.css|MediaType: text/css`) + b.AssertFileContent("public/index.html", `T2: Content: body{color:#333}|RelPermalink: /scss/styles2.min.css`) + b.AssertFileContent("public/index.html", `T3: Content: 24|RelPermalink: /styles/main.css|MediaType: text/css`) + b.AssertFileContent("public/index.html", `T4: Content: 24|RelPermalink: /styles/main.css|MediaType: text/css`) + b.AssertFileContent("public/index.html", `T5: Content: .content-navigation {`) + b.AssertFileContent("public/index.html", `T5 RelPermalink: /sass/styles3.css|`) + + }}, + + {"minify", func() bool { return true }, func(b *sitesBuilder) { + b.WithTemplates("home.html", ` +Min CSS: {{ ( resources.Get "css/styles1.css" | minify ).Content }} +Min JS: {{ ( resources.Get "js/script1.js" | resources.Minify ).Content | safeJS }} +Min JSON: {{ ( resources.Get "mydata/json1.json" | resources.Minify ).Content | safeHTML }} +Min XML: {{ ( resources.Get "mydata/xml1.xml" | resources.Minify ).Content | safeHTML }} +Min SVG: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }} +Min SVG again: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }} +Min HTML: {{ ( resources.Get "mydata/html1.html" | resources.Minify ).Content | safeHTML }} + + +`) + }, func(b *sitesBuilder) { + b.AssertFileContent("public/index.html", `Min CSS: h1{font-style:bold}`) + b.AssertFileContent("public/index.html", `Min JS: var x;x=5;document.getElementById("demo").innerHTML=x*10;`) + b.AssertFileContent("public/index.html", `Min JSON: {"employees":[{"firstName":"John","lastName":"Doe"},{"firstName":"Anna","lastName":"Smith"},{"firstName":"Peter","lastName":"Jones"}]}`) + b.AssertFileContent("public/index.html", `Min XML: <hello><world>Hugo Rocks!</<world></hello>`) + b.AssertFileContent("public/index.html", `Min SVG: <svg height="100" width="100"><path d="M5 10 20 40z"/></svg>`) + b.AssertFileContent("public/index.html", `Min SVG again: <svg height="100" width="100"><path d="M5 10 20 40z"/></svg>`) + b.AssertFileContent("public/index.html", `Min HTML: <a href=#>Cool</a>`) + }}, + + {"concat", func() bool { return true }, func(b *sitesBuilder) { + b.WithTemplates("home.html", ` +{{ $a := "A" | resources.FromString "a.txt"}} +{{ $b := "B" | resources.FromString "b.txt"}} +{{ $c := "C" | resources.FromString "c.txt"}} +{{ $combined := slice $a $b $c | resources.Concat "bundle/concat.txt" }} +T: Content: {{ $combined.Content }}|RelPermalink: {{ $combined.RelPermalink }}|Permalink: {{ $combined.Permalink }}|MediaType: {{ $combined.MediaType.Type }} +`) + }, func(b *sitesBuilder) { + b.AssertFileContent("public/index.html", `T: Content: ABC|RelPermalink: /bundle/concat.txt|Permalink: http://example.com/bundle/concat.txt|MediaType: text/plain`) + b.AssertFileContent("public/bundle/concat.txt", "ABC") + }}, + + {"fromstring", func() bool { return true }, func(b *sitesBuilder) { + b.WithTemplates("home.html", ` +{{ $r := "Hugo Rocks!" | resources.FromString "rocks/hugo.txt" }} +{{ $r.Content }}|{{ $r.RelPermalink }}|{{ $r.Permalink }}|{{ $r.MediaType.Type }} +`) + + }, func(b *sitesBuilder) { + b.AssertFileContent("public/index.html", `Hugo Rocks!|/rocks/hugo.txt|http://example.com/rocks/hugo.txt|text/plain`) + b.AssertFileContent("public/rocks/hugo.txt", "Hugo Rocks!") + + }}, + {"execute-as-template", func() bool { return true }, func(b *sitesBuilder) { + b.WithTemplates("home.html", ` + +{{ $result := "{{ .Kind | upper }}" | resources.FromString "mytpl.txt" | resources.ExecuteAsTemplate "result.txt" . }} +T1: {{ $result.Content }}|{{ $result.RelPermalink}}|{{$result.MediaType.Type }} +`) + + }, func(b *sitesBuilder) { + b.AssertFileContent("public/index.html", `T1: HOME|/result.txt|text/plain`) + + }}, + {"fingerprint", func() bool { return true }, func(b *sitesBuilder) { + b.WithTemplates("home.html", ` +{{ $r := "ab" | resources.FromString "rocks/hugo.txt" }} +{{ $result := $r | fingerprint }} +{{ $result512 := $r | fingerprint "sha512" }} +{{ $resultMD5 := $r | fingerprint "md5" }} +T1: {{ $result.Content }}|{{ $result.RelPermalink}}|{{$result.MediaType.Type }}|{{ $result.Data.Integrity }}| +T2: {{ $result512.Content }}|{{ $result512.RelPermalink}}|{{$result512.MediaType.Type }}|{{ $result512.Data.Integrity }}| +T3: {{ $resultMD5.Content }}|{{ $resultMD5.RelPermalink}}|{{$resultMD5.MediaType.Type }}|{{ $resultMD5.Data.Integrity }}| +`) + }, func(b *sitesBuilder) { + b.AssertFileContent("public/index.html", `T1: ab|/rocks/hugo.fb8e20fc2e4c3f248c60c39bd652f3c1347298bb977b8b4d5903b85055620603.txt|text/plain|sha256-+44g/C5MPySMYMOb1lLzwTRymLuXe4tNWQO4UFViBgM=|`) + b.AssertFileContent("public/index.html", `T2: ab|/rocks/hugo.2d408a0717ec188158278a796c689044361dc6fdde28d6f04973b80896e1823975cdbf12eb63f9e0591328ee235d80e9b5bf1aa6a44f4617ff3caf6400eb172d.txt|text/plain|sha512-LUCKBxfsGIFYJ4p5bGiQRDYdxv3eKNbwSXO4CJbhgjl1zb8S62P54FkTKO4jXYDptb8apqRPRhf/PK9kAOsXLQ==|`) + b.AssertFileContent("public/index.html", `T3: ab|/rocks/hugo.187ef4436122d1cc2f40dc2b92f0eba0.txt|text/plain|md5-GH70Q2Ei0cwvQNwrkvDroA==|`) + }}, + {"template", func() bool { return true }, func(b *sitesBuilder) {}, func(b *sitesBuilder) { + }}, + } + + for _, test := range tests { + if !test.shouldRun() { + t.Log("Skip", test.name) + continue + } + + b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger()) + b.WithSimpleConfigFile() + b.WithContent("page.md", ` +--- +title: Hello +--- + +`) + + b.WithSourceFile(filepath.Join("assets", "css", "styles1.css"), ` +h1 { + font-style: bold; +} +`) + + b.WithSourceFile(filepath.Join("assets", "js", "script1.js"), ` +var x; +x = 5; +document.getElementById("demo").innerHTML = x * 10; +`) + + b.WithSourceFile(filepath.Join("assets", "mydata", "json1.json"), ` +{ +"employees":[ + {"firstName":"John", "lastName":"Doe"}, + {"firstName":"Anna", "lastName":"Smith"}, + {"firstName":"Peter", "lastName":"Jones"} +] +} +`) + + b.WithSourceFile(filepath.Join("assets", "mydata", "svg1.svg"), ` +<svg height="100" width="100"> + <line x1="5" y1="10" x2="20" y2="40"/> +</svg> +`) + + b.WithSourceFile(filepath.Join("assets", "mydata", "xml1.xml"), ` +<hello> +<world>Hugo Rocks!</<world> +</hello> +`) + + b.WithSourceFile(filepath.Join("assets", "mydata", "html1.html"), ` +<html> +<a href="#"> +Cool +</a > +</html> +`) + + b.WithSourceFile(filepath.Join("assets", "scss", "styles2.scss"), ` +$color: #333; + +body { + color: $color; +} +`) + + b.WithSourceFile(filepath.Join("assets", "sass", "styles3.sass"), ` +$color: #333; + +.content-navigation + border-color: $color + +`) + + t.Log("Test", test.name) + test.prepare(b) + b.Build(BuildCfg{}) + test.verify(b) + } +} diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index bbd34e22b..c07a5586a 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -545,7 +545,7 @@ Loop: } var err error - isInner, err = isInnerShortcode(tmpl) + isInner, err = isInnerShortcode(tmpl.(tpl.TemplateExecutor)) if err != nil { return sc, fmt.Errorf("Failed to handle template for shortcode %q for page %q: %s", sc.name, p.Path(), err) } @@ -709,7 +709,7 @@ func replaceShortcodeTokens(source []byte, prefix string, replacements map[strin return source, nil } -func getShortcodeTemplateForTemplateKey(key scKey, shortcodeName string, t tpl.TemplateFinder) *tpl.TemplateAdapter { +func getShortcodeTemplateForTemplateKey(key scKey, shortcodeName string, t tpl.TemplateFinder) tpl.Template { isInnerShortcodeCache.RLock() defer isInnerShortcodeCache.RUnlock() @@ -737,13 +737,13 @@ func getShortcodeTemplateForTemplateKey(key scKey, shortcodeName string, t tpl.T for _, name := range names { - if x := t.Lookup("shortcodes/" + name); x != nil { + if x, found := t.Lookup("shortcodes/" + name); found { return x } - if x := t.Lookup("theme/shortcodes/" + name); x != nil { + if x, found := t.Lookup("theme/shortcodes/" + name); found { return x } - if x := t.Lookup("_internal/shortcodes/" + name); x != nil { + if x, found := t.Lookup("_internal/shortcodes/" + name); found { return x } } diff --git a/hugolib/site.go b/hugolib/site.go index 831216779..df7e66d4a 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -27,12 +27,12 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/resource" + "github.com/gohugoio/hugo/langs" src "github.com/gohugoio/hugo/source" - "github.com/gohugoio/hugo/resource" - "golang.org/x/sync/errgroup" "github.com/gohugoio/hugo/config" @@ -140,8 +140,7 @@ type Site struct { renderFormats output.Formats // Logger etc. - *deps.Deps `json:"-"` - resourceSpec *resource.Spec + *deps.Deps `json:"-"` // The func used to title case titles. titleFunc func(s string) string @@ -188,7 +187,6 @@ func (s *Site) reset() *Site { outputFormatsConfig: s.outputFormatsConfig, frontmatterHandler: s.frontmatterHandler, mediaTypesConfig: s.mediaTypesConfig, - resourceSpec: s.resourceSpec, Language: s.Language, owner: s.owner, PageCollections: newPageCollections()} @@ -691,7 +689,11 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { logger = helpers.NewDistinctFeedbackLogger() ) - for _, ev := range events { + cachePartitions := make([]string, len(events)) + + for i, ev := range events { + cachePartitions[i] = resource.ResourceKeyPartition(ev.Name) + if s.isContentDirEvent(ev) { logger.Println("Source changed", ev) sourceChanged = append(sourceChanged, ev) @@ -717,6 +719,11 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { } } + // These in memory resource caches will be rebuilt on demand. + for _, s := range s.owner.Sites { + s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...) + } + if len(tmplChanged) > 0 || len(i18nChanged) > 0 { sites := s.owner.Sites first := sites[0] @@ -731,7 +738,11 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { for i := 1; i < len(sites); i++ { site := sites[i] var err error - site.Deps, err = first.Deps.ForLanguage(site.Language) + depsCfg := deps.DepsCfg{ + Language: site.Language, + MediaTypes: site.mediaTypesConfig, + } + site.Deps, err = first.Deps.ForLanguage(depsCfg) if err != nil { return whatChanged{}, err } @@ -797,6 +808,7 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { if err := s.readAndProcessContent(filenamesChanged...); err != nil { return whatChanged{}, err } + } changed := whatChanged{ @@ -1240,7 +1252,7 @@ func (s *Site) readAndProcessContent(filenames ...string) error { mainHandler := &contentCaptureResultHandler{contentProcessors: contentProcessors, defaultContentProcessor: defaultContentProcessor} - sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.ContentFs) + sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.Content.Fs) if s.running() { // Need to track changes. @@ -1717,6 +1729,8 @@ func (s *Site) renderForLayouts(name string, d interface{}, w io.Writer, layouts templName = templ.Name() } s.DistinctErrorLog.Printf("Failed to render %q: %s", templName, r) + s.DistinctErrorLog.Printf("Stack Trace:\n%s", stackTrace(1200)) + // TOD(bep) we really need to fix this. Also see below. if !s.running() && !testMode { os.Exit(-1) @@ -1753,7 +1767,7 @@ func (s *Site) renderForLayouts(name string, d interface{}, w io.Writer, layouts func (s *Site) findFirstTemplate(layouts ...string) tpl.Template { for _, layout := range layouts { - if templ := s.Tmpl.Lookup(layout); templ != nil { + if templ, found := s.Tmpl.Lookup(layout); found { return templ } } @@ -1782,7 +1796,7 @@ func (s *Site) newNodePage(typ string, sections ...string) *Page { pageContentInit: &pageContentInit{}, Kind: typ, Source: Source{File: &source.FileInfo{}}, - Data: make(map[string]interface{}), + data: make(map[string]interface{}), Site: &s.Info, sections: sections, s: s} @@ -1797,7 +1811,7 @@ func (s *Site) newHomePage() *Page { p := s.newNodePage(KindHome) p.title = s.Info.Title pages := Pages{} - p.Data["Pages"] = pages + p.data["Pages"] = pages p.Pages = pages return p } diff --git a/hugolib/site_render.go b/hugolib/site_render.go index e837d9f0b..5efe6badc 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -252,7 +252,7 @@ func (s *Site) renderRSS(p *PageOutput) error { limit := s.Cfg.GetInt("rssLimit") if limit >= 0 && len(p.Pages) > limit { p.Pages = p.Pages[:limit] - p.Data["Pages"] = p.Pages + p.data["Pages"] = p.Pages } layouts, err := s.layoutHandler.For( @@ -279,7 +279,7 @@ func (s *Site) render404() error { p := s.newNodePage(kind404) p.title = "404 Page not found" - p.Data["Pages"] = s.Pages + p.data["Pages"] = s.Pages p.Pages = s.Pages p.URLPath.URL = "404.html" @@ -326,7 +326,7 @@ func (s *Site) renderSitemap() error { page.Sitemap.Priority = sitemapDefault.Priority page.Sitemap.Filename = sitemapDefault.Filename - n.Data["Pages"] = pages + n.data["Pages"] = pages n.Pages = pages // TODO(bep) we have several of these @@ -369,7 +369,7 @@ func (s *Site) renderRobotsTXT() error { if err := p.initTargetPathDescriptor(); err != nil { return err } - p.Data["Pages"] = s.Pages + p.data["Pages"] = s.Pages p.Pages = s.Pages rLayouts := []string{"robots.txt", "_default/robots.txt", "_internal/_default/robots.txt"} diff --git a/hugolib/site_sections.go b/hugolib/site_sections.go index 2537b5d24..2a92a3424 100644 --- a/hugolib/site_sections.go +++ b/hugolib/site_sections.go @@ -357,6 +357,6 @@ func (s *Site) assembleSections() Pages { func (p *Page) setPagePages(pages Pages) { pages.Sort() p.Pages = pages - p.Data = make(map[string]interface{}) - p.Data["Pages"] = pages + p.data = make(map[string]interface{}) + p.data["Pages"] = pages } diff --git a/hugolib/site_sections_test.go b/hugolib/site_sections_test.go index 9a75f65f9..01550c9fa 100644 --- a/hugolib/site_sections_test.go +++ b/hugolib/site_sections_test.go @@ -277,7 +277,7 @@ PAG|{{ .Title }}|{{ $sect.InSection . }} assert.NotNil(p, fmt.Sprint(sections)) if p.Pages != nil { - assert.Equal(p.Pages, p.Data["Pages"]) + assert.Equal(p.Pages, p.data["Pages"]) } assert.NotNil(p.Parent(), fmt.Sprintf("Parent nil: %q", test.sections)) test.verify(p) diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 93ea5032e..9fe60c434 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -441,7 +441,7 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { content := readDestination(s.T, s.Fs, filename) for _, match := range matches { if !strings.Contains(content, match) { - s.Fatalf("No match for %q in content for %s\n%s", match, filename, content) + s.Fatalf("No match for %q in content for %s\n%s\n%q", match, filename, content, content) } } } @@ -519,7 +519,7 @@ func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *helpers.PathSpec { return ps } -func newTestDefaultPathSpec() *helpers.PathSpec { +func newTestDefaultPathSpec(t *testing.T) *helpers.PathSpec { v := viper.New() // Easier to reason about in tests. v.Set("disablePathToLower", true) @@ -528,8 +528,14 @@ func newTestDefaultPathSpec() *helpers.PathSpec { v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") v.Set("archetypeDir", "archetypes") + v.Set("assetDir", "assets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") fs := hugofs.NewDefault(v) - ps, _ := helpers.NewPathSpec(fs, v) + ps, err := helpers.NewPathSpec(fs, v) + if err != nil { + t.Fatal(err) + } return ps } diff --git a/i18n/i18n_test.go b/i18n/i18n_test.go index c5c962c16..5075839ff 100644 --- a/i18n/i18n_test.go +++ b/i18n/i18n_test.go @@ -205,6 +205,9 @@ func TestI18nTranslate(t *testing.T) { v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") v.Set("archetypeDir", "archetypes") + v.Set("assetDir", "assets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") // Test without and with placeholders for _, enablePlaceholders := range []bool{false, true} { diff --git a/magefile.go b/magefile.go index 0cede2697..e33816511 100644 --- a/magefile.go +++ b/magefile.go @@ -46,17 +46,17 @@ func Vendor() error { // Build hugo binary func Hugo() error { - return sh.RunWith(flagEnv(), goexe, "build", "-ldflags", ldflags, packageName) + return sh.RunWith(flagEnv(), goexe, "build", "-ldflags", ldflags, "-tags", buildTags(), packageName) } // Build hugo binary with race detector enabled func HugoRace() error { - return sh.RunWith(flagEnv(), goexe, "build", "-race", "-ldflags", ldflags, packageName) + return sh.RunWith(flagEnv(), goexe, "build", "-race", "-ldflags", ldflags, "-tags", buildTags(), packageName) } // Install hugo binary func Install() error { - return sh.RunWith(flagEnv(), goexe, "install", "-ldflags", ldflags, packageName) + return sh.RunWith(flagEnv(), goexe, "install", "-ldflags", ldflags, "-tags", buildTags(), packageName) } func flagEnv() map[string]string { @@ -111,18 +111,19 @@ func Check() { } // Run tests in 32-bit mode +// Note that we don't run with the extended tag. Currently not supported in 32 bit. func Test386() error { return sh.RunWith(map[string]string{"GOARCH": "386"}, goexe, "test", "./...") } // Run tests func Test() error { - return sh.Run(goexe, "test", "./...") + return sh.Run(goexe, "test", "./...", "-tags", buildTags()) } // Run tests with race detector func TestRace() error { - return sh.Run(goexe, "test", "-race", "./...") + return sh.Run(goexe, "test", "-race", "./...", "-tags", buildTags()) } // Run gofmt linter @@ -266,3 +267,13 @@ func CheckVendor() error { func isGoLatest() bool { return strings.Contains(runtime.Version(), "1.10") } + +func buildTags() string { + // To build the extended Hugo SCSS/SASS enabled version, build with + // HUGO_BUILD_TAGS=extended mage install etc. + if envtags := os.Getenv("HUGO_BUILD_TAGS"); envtags != "" { + return envtags + } + return "none" + +} diff --git a/media/mediaType.go b/media/mediaType.go index 33ccb2818..07ba410fb 100644 --- a/media/mediaType.go +++ b/media/mediaType.go @@ -50,7 +50,8 @@ func FromString(t string) (Type, error) { mainType := parts[0] subParts := strings.Split(parts[1], "+") - subType := subParts[0] + subType := strings.Split(subParts[0], ";")[0] + var suffix string if len(subParts) == 1 { @@ -85,25 +86,38 @@ func (m Type) FullSuffix() string { var ( CalendarType = Type{"text", "calendar", "ics", defaultDelimiter} CSSType = Type{"text", "css", "css", defaultDelimiter} + SCSSType = Type{"text", "x-scss", "scss", defaultDelimiter} + SASSType = Type{"text", "x-sass", "sass", defaultDelimiter} CSVType = Type{"text", "csv", "csv", defaultDelimiter} HTMLType = Type{"text", "html", "html", defaultDelimiter} JavascriptType = Type{"application", "javascript", "js", defaultDelimiter} JSONType = Type{"application", "json", "json", defaultDelimiter} RSSType = Type{"application", "rss", "xml", defaultDelimiter} XMLType = Type{"application", "xml", "xml", defaultDelimiter} - TextType = Type{"text", "plain", "txt", defaultDelimiter} + // The official MIME type of SVG is image/svg+xml. We currently only support one extension + // per mime type. The workaround in projects is to create multiple media type definitions, + // but we need to improve this to take other known suffixes into account. + // But until then, svg has an svg extension, which is very common. TODO(bep) + SVGType = Type{"image", "svg", "svg", defaultDelimiter} + TextType = Type{"text", "plain", "txt", defaultDelimiter} + + OctetType = Type{"application", "octet-stream", "", ""} ) var DefaultTypes = Types{ CalendarType, CSSType, CSVType, + SCSSType, + SASSType, HTMLType, JavascriptType, JSONType, RSSType, XMLType, + SVGType, TextType, + OctetType, } func init() { @@ -125,6 +139,16 @@ func (t Types) GetByType(tp string) (Type, bool) { return Type{}, false } +// GetFirstBySuffix will return the first media type matching the given suffix. +func (t Types) GetFirstBySuffix(suffix string) (Type, bool) { + for _, tt := range t { + if strings.EqualFold(suffix, tt.Suffix) { + return tt, true + } + } + return Type{}, false +} + // GetBySuffix gets a media type given as suffix, e.g. "html". // It will return false if no format could be found, or if the suffix given // is ambiguous. diff --git a/media/mediaType_test.go b/media/mediaType_test.go index 0cdecdeb1..f3ddb086c 100644 --- a/media/mediaType_test.go +++ b/media/mediaType_test.go @@ -30,12 +30,15 @@ func TestDefaultTypes(t *testing.T) { }{ {CalendarType, "text", "calendar", "ics", "text/calendar", "text/calendar+ics"}, {CSSType, "text", "css", "css", "text/css", "text/css+css"}, + {SCSSType, "text", "x-scss", "scss", "text/x-scss", "text/x-scss+scss"}, {CSVType, "text", "csv", "csv", "text/csv", "text/csv+csv"}, {HTMLType, "text", "html", "html", "text/html", "text/html+html"}, {JavascriptType, "application", "javascript", "js", "application/javascript", "application/javascript+js"}, {JSONType, "application", "json", "json", "application/json", "application/json+json"}, {RSSType, "application", "rss", "xml", "application/rss", "application/rss+xml"}, + {SVGType, "image", "svg", "svg", "image/svg", "image/svg+svg"}, {TextType, "text", "plain", "txt", "text/plain", "text/plain+txt"}, + {XMLType, "application", "xml", "xml", "application/xml", "application/xml+xml"}, } { require.Equal(t, test.expectedMainType, test.tp.MainType) require.Equal(t, test.expectedSubType, test.tp.SubType) @@ -60,6 +63,13 @@ func TestGetByType(t *testing.T) { require.False(t, found) } +func TestGetFirstBySuffix(t *testing.T) { + assert := require.New(t) + f, found := DefaultTypes.GetFirstBySuffix("xml") + assert.True(found) + assert.Equal(Type{MainType: "application", SubType: "rss", Suffix: "xml", Delimiter: "."}, f) +} + func TestFromTypeString(t *testing.T) { f, err := FromString("text/html") require.NoError(t, err) @@ -76,6 +86,10 @@ func TestFromTypeString(t *testing.T) { _, err = FromString("noslash") require.Error(t, err) + f, err = FromString("text/xml; charset=utf-8") + require.NoError(t, err) + require.Equal(t, Type{MainType: "text", SubType: "xml", Suffix: "xml", Delimiter: "."}, f) + } func TestDecodeTypes(t *testing.T) { diff --git a/parser/long_text_test.md b/parser/long_text_test.md index e87ceb8c6..e0cac502c 100644 --- a/parser/long_text_test.md +++ b/parser/long_text_test.md @@ -1,263 +1,263 @@ ----
-title: The Git Book - Long Text
----
-# Getting Started #
-
-This chapter will be about getting started with Git. We will begin at the beginning by explaining some background on version control tools, then move on to how to get Git running on your system and finally how to get it setup to start working with. At the end of this chapter you should understand why Git is around, why you should use it and you should be all setup to do so.
-
-## About Version Control ##
-
-What is version control, and why should you care? Version control is a system that records changes to a file or set of files over time so that you can recall specific versions later. Even though the examples in this book show software source code as the files under version control, in reality any type of file on a computer can be placed under version control.
-
-If you are a graphic or web designer and want to keep every version of an image or layout (which you certainly would), it is very wise to use a Version Control System (VCS). A VCS allows you to: revert files back to a previous state, revert the entire project back to a previous state, review changes made over time, see who last modified something that might be causing a problem, who introduced an issue and when, and more. Using a VCS also means that if you screw things up or lose files, you can generally recover easily. In addition, you get all this for very little overhead.
-
-### Local Version Control Systems ###
-
-Many people�s version-control method of choice is to copy files into another directory (perhaps a time-stamped directory, if they�re clever). This approach is very common because it is so simple, but it is also incredibly error prone. It is easy to forget which directory you�re in and accidentally write to the wrong file or copy over files you don�t mean to.
-
-To deal with this issue, programmers long ago developed local VCSs that had a simple database that kept all the changes to files under revision control (see Figure 1-1).
-
-Insert 18333fig0101.png
-Figure 1-1. Local version control diagram.
-
-One of the more popular VCS tools was a system called rcs, which is still distributed with many computers today. Even the popular Mac OS X operating system includes the rcs command when you install the Developer Tools. This tool basically works by keeping patch sets (that is, the differences between files) from one revision to another in a special format on disk; it can then recreate what any file looked like at any point in time by adding up all the patches.
-
-### Centralized Version Control Systems ###
-
-The next major issue that people encounter is that they need to collaborate with developers on other systems. To deal with this problem, Centralized Version Control Systems (CVCSs) were developed. These systems, such as CVS, Subversion, and Perforce, have a single server that contains all the versioned files, and a number of clients that check out files from that central place. For many years, this has been the standard for version control (see Figure 1-2).
-
-Insert 18333fig0102.png
-Figure 1-2. Centralized version control diagram.
-
-This setup offers many advantages, especially over local VCSs. For example, everyone knows to a certain degree what everyone else on the project is doing. Administrators have fine-grained control over who can do what; and it�s far easier to administer a CVCS than it is to deal with local databases on every client.
-
-However, this setup also has some serious downsides. The most obvious is the single point of failure that the centralized server represents. If that server goes down for an hour, then during that hour nobody can collaborate at all or save versioned changes to anything they�re working on. If the hard disk the central database is on becomes corrupted, and proper backups haven�t been kept, you lose absolutely everything�the entire history of the project except whatever single snapshots people happen to have on their local machines. Local VCS systems suffer from this same problem�whenever you have the entire history of the project in a single place, you risk losing everything.
-
-### Distributed Version Control Systems ###
-
-This is where Distributed Version Control Systems (DVCSs) step in. In a DVCS (such as Git, Mercurial, Bazaar or Darcs), clients don�t just check out the latest snapshot of the files: they fully mirror the repository. Thus if any server dies, and these systems were collaborating via it, any of the client repositories can be copied back up to the server to restore it. Every checkout is really a full backup of all the data (see Figure 1-3).
-
-Insert 18333fig0103.png
-Figure 1-3. Distributed version control diagram.
-
-Furthermore, many of these systems deal pretty well with having several remote repositories they can work with, so you can collaborate with different groups of people in different ways simultaneously within the same project. This allows you to set up several types of workflows that aren�t possible in centralized systems, such as hierarchical models.
-
-## A Short History of Git ##
-
-As with many great things in life, Git began with a bit of creative destruction and fiery controversy. The Linux kernel is an open source software project of fairly large scope. For most of the lifetime of the Linux kernel maintenance (1991�2002), changes to the software were passed around as patches and archived files. In 2002, the Linux kernel project began using a proprietary DVCS system called BitKeeper.
-
-In 2005, the relationship between the community that developed the Linux kernel and the commercial company that developed BitKeeper broke down, and the tool�s free-of-charge status was revoked. This prompted the Linux development community (and in particular Linus Torvalds, the creator of Linux) to develop their own tool based on some of the lessons they learned while using BitKeeper. Some of the goals of the new system were as follows:
-
-* Speed
-* Simple design
-* Strong support for non-linear development (thousands of parallel branches)
-* Fully distributed
-* Able to handle large projects like the Linux kernel efficiently (speed and data size)
-
-Since its birth in 2005, Git has evolved and matured to be easy to use and yet retain these initial qualities. It�s incredibly fast, it�s very efficient with large projects, and it has an incredible branching system for non-linear development (See Chapter 3).
-
-## Git Basics ##
-
-So, what is Git in a nutshell? This is an important section to absorb, because if you understand what Git is and the fundamentals of how it works, then using Git effectively will probably be much easier for you. As you learn Git, try to clear your mind of the things you may know about other VCSs, such as Subversion and Perforce; doing so will help you avoid subtle confusion when using the tool. Git stores and thinks about information much differently than these other systems, even though the user interface is fairly similar; understanding those differences will help prevent you from becoming confused while using it.
-
-### Snapshots, Not Differences ###
-
-The major difference between Git and any other VCS (Subversion and friends included) is the way Git thinks about its data. Conceptually, most other systems store information as a list of file-based changes. These systems (CVS, Subversion, Perforce, Bazaar, and so on) think of the information they keep as a set of files and the changes made to each file over time, as illustrated in Figure 1-4.
-
-Insert 18333fig0104.png
-Figure 1-4. Other systems tend to store data as changes to a base version of each file.
-
-Git doesn�t think of or store its data this way. Instead, Git thinks of its data more like a set of snapshots of a mini filesystem. Every time you commit, or save the state of your project in Git, it basically takes a picture of what all your files look like at that moment and stores a reference to that snapshot. To be efficient, if files have not changed, Git doesn�t store the file again�just a link to the previous identical file it has already stored. Git thinks about its data more like Figure 1-5.
-
-Insert 18333fig0105.png
-Figure 1-5. Git stores data as snapshots of the project over time.
-
-This is an important distinction between Git and nearly all other VCSs. It makes Git reconsider almost every aspect of version control that most other systems copied from the previous generation. This makes Git more like a mini filesystem with some incredibly powerful tools built on top of it, rather than simply a VCS. We�ll explore some of the benefits you gain by thinking of your data this way when we cover Git branching in Chapter 3.
-
-### Nearly Every Operation Is Local ###
-
-Most operations in Git only need local files and resources to operate � generally no information is needed from another computer on your network. If you�re used to a CVCS where most operations have that network latency overhead, this aspect of Git will make you think that the gods of speed have blessed Git with unworldly powers. Because you have the entire history of the project right there on your local disk, most operations seem almost instantaneous.
-
-For example, to browse the history of the project, Git doesn�t need to go out to the server to get the history and display it for you�it simply reads it directly from your local database. This means you see the project history almost instantly. If you want to see the changes introduced between the current version of a file and the file a month ago, Git can look up the file a month ago and do a local difference calculation, instead of having to either ask a remote server to do it or pull an older version of the file from the remote server to do it locally.
-
-This also means that there is very little you can�t do if you�re offline or off VPN. If you get on an airplane or a train and want to do a little work, you can commit happily until you get to a network connection to upload. If you go home and can�t get your VPN client working properly, you can still work. In many other systems, doing so is either impossible or painful. In Perforce, for example, you can�t do much when you aren�t connected to the server; and in Subversion and CVS, you can edit files, but you can�t commit changes to your database (because your database is offline). This may not seem like a huge deal, but you may be surprised what a big difference it can make.
-
-### Git Has Integrity ###
-
-Everything in Git is check-summed before it is stored and is then referred to by that checksum. This means it�s impossible to change the contents of any file or directory without Git knowing about it. This functionality is built into Git at the lowest levels and is integral to its philosophy. You can�t lose information in transit or get file corruption without Git being able to detect it.
-
-The mechanism that Git uses for this checksumming is called a SHA-1 hash. This is a 40-character string composed of hexadecimal characters (0�9 and a�f) and calculated based on the contents of a file or directory structure in Git. A SHA-1 hash looks something like this:
-
- 24b9da6552252987aa493b52f8696cd6d3b00373
-
-You will see these hash values all over the place in Git because it uses them so much. In fact, Git stores everything not by file name but in the Git database addressable by the hash value of its contents.
-
-### Git Generally Only Adds Data ###
-
-When you do actions in Git, nearly all of them only add data to the Git database. It is very difficult to get the system to do anything that is not undoable or to make it erase data in any way. As in any VCS, you can lose or mess up changes you haven�t committed yet; but after you commit a snapshot into Git, it is very difficult to lose, especially if you regularly push your database to another repository.
-
-This makes using Git a joy because we know we can experiment without the danger of severely screwing things up. For a more in-depth look at how Git stores its data and how you can recover data that seems lost, see Chapter 9.
-
-### The Three States ###
-
-Now, pay attention. This is the main thing to remember about Git if you want the rest of your learning process to go smoothly. Git has three main states that your files can reside in: committed, modified, and staged. Committed means that the data is safely stored in your local database. Modified means that you have changed the file but have not committed it to your database yet. Staged means that you have marked a modified file in its current version to go into your next commit snapshot.
-
-This leads us to the three main sections of a Git project: the Git directory, the working directory, and the staging area.
-
-Insert 18333fig0106.png
-Figure 1-6. Working directory, staging area, and git directory.
-
-The Git directory is where Git stores the metadata and object database for your project. This is the most important part of Git, and it is what is copied when you clone a repository from another computer.
-
-The working directory is a single checkout of one version of the project. These files are pulled out of the compressed database in the Git directory and placed on disk for you to use or modify.
-
-The staging area is a simple file, generally contained in your Git directory, that stores information about what will go into your next commit. It�s sometimes referred to as the index, but it�s becoming standard to refer to it as the staging area.
-
-The basic Git workflow goes something like this:
-
-1. You modify files in your working directory.
-2. You stage the files, adding snapshots of them to your staging area.
-3. You do a commit, which takes the files as they are in the staging area and stores that snapshot permanently to your Git directory.
-
-If a particular version of a file is in the git directory, it�s considered committed. If it�s modified but has been added to the staging area, it is staged. And if it was changed since it was checked out but has not been staged, it is modified. In Chapter 2, you�ll learn more about these states and how you can either take advantage of them or skip the staged part entirely.
-
-## Installing Git ##
-
-Let�s get into using some Git. First things first�you have to install it. You can get it a number of ways; the two major ones are to install it from source or to install an existing package for your platform.
-
-### Installing from Source ###
-
-If you can, it�s generally useful to install Git from source, because you�ll get the most recent version. Each version of Git tends to include useful UI enhancements, so getting the latest version is often the best route if you feel comfortable compiling software from source. It is also the case that many Linux distributions contain very old packages; so unless you�re on a very up-to-date distro or are using backports, installing from source may be the best bet.
-
-To install Git, you need to have the following libraries that Git depends on: curl, zlib, openssl, expat, and libiconv. For example, if you�re on a system that has yum (such as Fedora) or apt-get (such as a Debian based system), you can use one of these commands to install all of the dependencies:
-
- $ yum install curl-devel expat-devel gettext-devel \
- openssl-devel zlib-devel
-
- $ apt-get install libcurl4-gnutls-dev libexpat1-dev gettext \
- libz-dev libssl-dev
-
-When you have all the necessary dependencies, you can go ahead and grab the latest snapshot from the Git web site:
-
- http://git-scm.com/download
-
-Then, compile and install:
-
- $ tar -zxf git-1.7.2.2.tar.gz
- $ cd git-1.7.2.2
- $ make prefix=/usr/local all
- $ sudo make prefix=/usr/local install
-
-After this is done, you can also get Git via Git itself for updates:
-
- $ git clone git://git.kernel.org/pub/scm/git/git.git
-
-### Installing on Linux ###
-
-If you want to install Git on Linux via a binary installer, you can generally do so through the basic package-management tool that comes with your distribution. If you�re on Fedora, you can use yum:
-
- $ yum install git-core
-
-Or if you�re on a Debian-based distribution like Ubuntu, try apt-get:
-
- $ apt-get install git
-
-### Installing on Mac ###
-
-There are two easy ways to install Git on a Mac. The easiest is to use the graphical Git installer, which you can download from the Google Code page (see Figure 1-7):
-
- http://code.google.com/p/git-osx-installer
-
-Insert 18333fig0107.png
-Figure 1-7. Git OS X installer.
-
-The other major way is to install Git via MacPorts (`http://www.macports.org`). If you have MacPorts installed, install Git via
-
- $ sudo port install git-core +svn +doc +bash_completion +gitweb
-
-You don�t have to add all the extras, but you�ll probably want to include +svn in case you ever have to use Git with Subversion repositories (see Chapter 8).
-
-### Installing on Windows ###
-
-Installing Git on Windows is very easy. The msysGit project has one of the easier installation procedures. Simply download the installer exe file from the GitHub page, and run it:
-
- http://msysgit.github.com/
-
-After it�s installed, you have both a command-line version (including an SSH client that will come in handy later) and the standard GUI.
-
-Note on Windows usage: you should use Git with the provided msysGit shell (Unix style), it allows to use the complex lines of command given in this book. If you need, for some reason, to use the native Windows shell / command line console, you have to use double quotes instead of simple quotes (for parameters with spaces in them) and you must quote the parameters ending with the circumflex accent (^) if they are last on the line, as it is a continuation symbol in Windows.
-
-## First-Time Git Setup ##
-
-Now that you have Git on your system, you�ll want to do a few things to customize your Git environment. You should have to do these things only once; they�ll stick around between upgrades. You can also change them at any time by running through the commands again.
-
-Git comes with a tool called git config that lets you get and set configuration variables that control all aspects of how Git looks and operates. These variables can be stored in three different places:
-
-* `/etc/gitconfig` file: Contains values for every user on the system and all their repositories. If you pass the option` --system` to `git config`, it reads and writes from this file specifically.
-* `~/.gitconfig` file: Specific to your user. You can make Git read and write to this file specifically by passing the `--global` option.
-* config file in the git directory (that is, `.git/config`) of whatever repository you�re currently using: Specific to that single repository. Each level overrides values in the previous level, so values in `.git/config` trump those in `/etc/gitconfig`.
-
-On Windows systems, Git looks for the `.gitconfig` file in the `$HOME` directory (`%USERPROFILE%` in Windows� environment), which is `C:\Documents and Settings\$USER` or `C:\Users\$USER` for most people, depending on version (`$USER` is `%USERNAME%` in Windows� environment). It also still looks for /etc/gitconfig, although it�s relative to the MSys root, which is wherever you decide to install Git on your Windows system when you run the installer.
-
-### Your Identity ###
-
-The first thing you should do when you install Git is to set your user name and e-mail address. This is important because every Git commit uses this information, and it�s immutably baked into the commits you pass around:
-
- $ git config --global user.name "John Doe"
- $ git config --global user.email [email protected]
-
-Again, you need to do this only once if you pass the `--global` option, because then Git will always use that information for anything you do on that system. If you want to override this with a different name or e-mail address for specific projects, you can run the command without the `--global` option when you�re in that project.
-
-### Your Editor ###
-
-Now that your identity is set up, you can configure the default text editor that will be used when Git needs you to type in a message. By default, Git uses your system�s default editor, which is generally Vi or Vim. If you want to use a different text editor, such as Emacs, you can do the following:
-
- $ git config --global core.editor emacs
-
-### Your Diff Tool ###
-
-Another useful option you may want to configure is the default diff tool to use to resolve merge conflicts. Say you want to use vimdiff:
-
- $ git config --global merge.tool vimdiff
-
-Git accepts kdiff3, tkdiff, meld, xxdiff, emerge, vimdiff, gvimdiff, ecmerge, and opendiff as valid merge tools. You can also set up a custom tool; see Chapter 7 for more information about doing that.
-
-### Checking Your Settings ###
-
-If you want to check your settings, you can use the `git config --list` command to list all the settings Git can find at that point:
-
- $ git config --list
- user.name=Scott Chacon
- color.status=auto
- color.branch=auto
- color.interactive=auto
- color.diff=auto
- ...
-
-You may see keys more than once, because Git reads the same key from different files (`/etc/gitconfig` and `~/.gitconfig`, for example). In this case, Git uses the last value for each unique key it sees.
-
-You can also check what Git thinks a specific key�s value is by typing `git config {key}`:
-
- $ git config user.name
- Scott Chacon
-
-## Getting Help ##
-
-If you ever need help while using Git, there are three ways to get the manual page (manpage) help for any of the Git commands:
-
- $ git help <verb>
- $ git <verb> --help
- $ man git-<verb>
-
-For example, you can get the manpage help for the config command by running
-
- $ git help config
-
-These commands are nice because you can access them anywhere, even offline.
-If the manpages and this book aren�t enough and you need in-person help, you can try the `#git` or `#github` channel on the Freenode IRC server (irc.freenode.net). These channels are regularly filled with hundreds of people who are all very knowledgeable about Git and are often willing to help.
-
-## Summary ##
-
-You should have a basic understanding of what Git is and how it�s different from the CVCS you may have been using. You should also now have a working version of Git on your system that�s set up with your personal identity. It�s now time to learn some Git basics.
-
+--- +title: The Git Book - Long Text +--- +# Getting Started # + +This chapter will be about getting started with Git. We will begin at the beginning by explaining some background on version control tools, then move on to how to get Git running on your system and finally how to get it setup to start working with. At the end of this chapter you should understand why Git is around, why you should use it and you should be all setup to do so. + +## About Version Control ## + +What is version control, and why should you care? Version control is a system that records changes to a file or set of files over time so that you can recall specific versions later. Even though the examples in this book show software source code as the files under version control, in reality any type of file on a computer can be placed under version control. + +If you are a graphic or web designer and want to keep every version of an image or layout (which you certainly would), it is very wise to use a Version Control System (VCS). A VCS allows you to: revert files back to a previous state, revert the entire project back to a previous state, review changes made over time, see who last modified something that might be causing a problem, who introduced an issue and when, and more. Using a VCS also means that if you screw things up or lose files, you can generally recover easily. In addition, you get all this for very little overhead. + +### Local Version Control Systems ### + +Many people�s version-control method of choice is to copy files into another directory (perhaps a time-stamped directory, if they�re clever). This approach is very common because it is so simple, but it is also incredibly error prone. It is easy to forget which directory you�re in and accidentally write to the wrong file or copy over files you don�t mean to. + +To deal with this issue, programmers long ago developed local VCSs that had a simple database that kept all the changes to files under revision control (see Figure 1-1). + +Insert 18333fig0101.png +Figure 1-1. Local version control diagram. + +One of the more popular VCS tools was a system called rcs, which is still distributed with many computers today. Even the popular Mac OS X operating system includes the rcs command when you install the Developer Tools. This tool basically works by keeping patch sets (that is, the differences between files) from one revision to another in a special format on disk; it can then recreate what any file looked like at any point in time by adding up all the patches. + +### Centralized Version Control Systems ### + +The next major issue that people encounter is that they need to collaborate with developers on other systems. To deal with this problem, Centralized Version Control Systems (CVCSs) were developed. These systems, such as CVS, Subversion, and Perforce, have a single server that contains all the versioned files, and a number of clients that check out files from that central place. For many years, this has been the standard for version control (see Figure 1-2). + +Insert 18333fig0102.png +Figure 1-2. Centralized version control diagram. + +This setup offers many advantages, especially over local VCSs. For example, everyone knows to a certain degree what everyone else on the project is doing. Administrators have fine-grained control over who can do what; and it�s far easier to administer a CVCS than it is to deal with local databases on every client. + +However, this setup also has some serious downsides. The most obvious is the single point of failure that the centralized server represents. If that server goes down for an hour, then during that hour nobody can collaborate at all or save versioned changes to anything they�re working on. If the hard disk the central database is on becomes corrupted, and proper backups haven�t been kept, you lose absolutely everything�the entire history of the project except whatever single snapshots people happen to have on their local machines. Local VCS systems suffer from this same problem�whenever you have the entire history of the project in a single place, you risk losing everything. + +### Distributed Version Control Systems ### + +This is where Distributed Version Control Systems (DVCSs) step in. In a DVCS (such as Git, Mercurial, Bazaar or Darcs), clients don�t just check out the latest snapshot of the files: they fully mirror the repository. Thus if any server dies, and these systems were collaborating via it, any of the client repositories can be copied back up to the server to restore it. Every checkout is really a full backup of all the data (see Figure 1-3). + +Insert 18333fig0103.png +Figure 1-3. Distributed version control diagram. + +Furthermore, many of these systems deal pretty well with having several remote repositories they can work with, so you can collaborate with different groups of people in different ways simultaneously within the same project. This allows you to set up several types of workflows that aren�t possible in centralized systems, such as hierarchical models. + +## A Short History of Git ## + +As with many great things in life, Git began with a bit of creative destruction and fiery controversy. The Linux kernel is an open source software project of fairly large scope. For most of the lifetime of the Linux kernel maintenance (1991�2002), changes to the software were passed around as patches and archived files. In 2002, the Linux kernel project began using a proprietary DVCS system called BitKeeper. + +In 2005, the relationship between the community that developed the Linux kernel and the commercial company that developed BitKeeper broke down, and the tool�s free-of-charge status was revoked. This prompted the Linux development community (and in particular Linus Torvalds, the creator of Linux) to develop their own tool based on some of the lessons they learned while using BitKeeper. Some of the goals of the new system were as follows: + +* Speed +* Simple design +* Strong support for non-linear development (thousands of parallel branches) +* Fully distributed +* Able to handle large projects like the Linux kernel efficiently (speed and data size) + +Since its birth in 2005, Git has evolved and matured to be easy to use and yet retain these initial qualities. It�s incredibly fast, it�s very efficient with large projects, and it has an incredible branching system for non-linear development (See Chapter 3). + +## Git Basics ## + +So, what is Git in a nutshell? This is an important section to absorb, because if you understand what Git is and the fundamentals of how it works, then using Git effectively will probably be much easier for you. As you learn Git, try to clear your mind of the things you may know about other VCSs, such as Subversion and Perforce; doing so will help you avoid subtle confusion when using the tool. Git stores and thinks about information much differently than these other systems, even though the user interface is fairly similar; understanding those differences will help prevent you from becoming confused while using it. + +### Snapshots, Not Differences ### + +The major difference between Git and any other VCS (Subversion and friends included) is the way Git thinks about its data. Conceptually, most other systems store information as a list of file-based changes. These systems (CVS, Subversion, Perforce, Bazaar, and so on) think of the information they keep as a set of files and the changes made to each file over time, as illustrated in Figure 1-4. + +Insert 18333fig0104.png +Figure 1-4. Other systems tend to store data as changes to a base version of each file. + +Git doesn�t think of or store its data this way. Instead, Git thinks of its data more like a set of snapshots of a mini filesystem. Every time you commit, or save the state of your project in Git, it basically takes a picture of what all your files look like at that moment and stores a reference to that snapshot. To be efficient, if files have not changed, Git doesn�t store the file again�just a link to the previous identical file it has already stored. Git thinks about its data more like Figure 1-5. + +Insert 18333fig0105.png +Figure 1-5. Git stores data as snapshots of the project over time. + +This is an important distinction between Git and nearly all other VCSs. It makes Git reconsider almost every aspect of version control that most other systems copied from the previous generation. This makes Git more like a mini filesystem with some incredibly powerful tools built on top of it, rather than simply a VCS. We�ll explore some of the benefits you gain by thinking of your data this way when we cover Git branching in Chapter 3. + +### Nearly Every Operation Is Local ### + +Most operations in Git only need local files and resources to operate � generally no information is needed from another computer on your network. If you�re used to a CVCS where most operations have that network latency overhead, this aspect of Git will make you think that the gods of speed have blessed Git with unworldly powers. Because you have the entire history of the project right there on your local disk, most operations seem almost instantaneous. + +For example, to browse the history of the project, Git doesn�t need to go out to the server to get the history and display it for you�it simply reads it directly from your local database. This means you see the project history almost instantly. If you want to see the changes introduced between the current version of a file and the file a month ago, Git can look up the file a month ago and do a local difference calculation, instead of having to either ask a remote server to do it or pull an older version of the file from the remote server to do it locally. + +This also means that there is very little you can�t do if you�re offline or off VPN. If you get on an airplane or a train and want to do a little work, you can commit happily until you get to a network connection to upload. If you go home and can�t get your VPN client working properly, you can still work. In many other systems, doing so is either impossible or painful. In Perforce, for example, you can�t do much when you aren�t connected to the server; and in Subversion and CVS, you can edit files, but you can�t commit changes to your database (because your database is offline). This may not seem like a huge deal, but you may be surprised what a big difference it can make. + +### Git Has Integrity ### + +Everything in Git is check-summed before it is stored and is then referred to by that checksum. This means it�s impossible to change the contents of any file or directory without Git knowing about it. This functionality is built into Git at the lowest levels and is integral to its philosophy. You can�t lose information in transit or get file corruption without Git being able to detect it. + +The mechanism that Git uses for this checksumming is called a SHA-1 hash. This is a 40-character string composed of hexadecimal characters (0�9 and a�f) and calculated based on the contents of a file or directory structure in Git. A SHA-1 hash looks something like this: + + 24b9da6552252987aa493b52f8696cd6d3b00373 + +You will see these hash values all over the place in Git because it uses them so much. In fact, Git stores everything not by file name but in the Git database addressable by the hash value of its contents. + +### Git Generally Only Adds Data ### + +When you do actions in Git, nearly all of them only add data to the Git database. It is very difficult to get the system to do anything that is not undoable or to make it erase data in any way. As in any VCS, you can lose or mess up changes you haven�t committed yet; but after you commit a snapshot into Git, it is very difficult to lose, especially if you regularly push your database to another repository. + +This makes using Git a joy because we know we can experiment without the danger of severely screwing things up. For a more in-depth look at how Git stores its data and how you can recover data that seems lost, see Chapter 9. + +### The Three States ### + +Now, pay attention. This is the main thing to remember about Git if you want the rest of your learning process to go smoothly. Git has three main states that your files can reside in: committed, modified, and staged. Committed means that the data is safely stored in your local database. Modified means that you have changed the file but have not committed it to your database yet. Staged means that you have marked a modified file in its current version to go into your next commit snapshot. + +This leads us to the three main sections of a Git project: the Git directory, the working directory, and the staging area. + +Insert 18333fig0106.png +Figure 1-6. Working directory, staging area, and git directory. + +The Git directory is where Git stores the metadata and object database for your project. This is the most important part of Git, and it is what is copied when you clone a repository from another computer. + +The working directory is a single checkout of one version of the project. These files are pulled out of the compressed database in the Git directory and placed on disk for you to use or modify. + +The staging area is a simple file, generally contained in your Git directory, that stores information about what will go into your next commit. It�s sometimes referred to as the index, but it�s becoming standard to refer to it as the staging area. + +The basic Git workflow goes something like this: + +1. You modify files in your working directory. +2. You stage the files, adding snapshots of them to your staging area. +3. You do a commit, which takes the files as they are in the staging area and stores that snapshot permanently to your Git directory. + +If a particular version of a file is in the git directory, it�s considered committed. If it�s modified but has been added to the staging area, it is staged. And if it was changed since it was checked out but has not been staged, it is modified. In Chapter 2, you�ll learn more about these states and how you can either take advantage of them or skip the staged part entirely. + +## Installing Git ## + +Let�s get into using some Git. First things first�you have to install it. You can get it a number of ways; the two major ones are to install it from source or to install an existing package for your platform. + +### Installing from Source ### + +If you can, it�s generally useful to install Git from source, because you�ll get the most recent version. Each version of Git tends to include useful UI enhancements, so getting the latest version is often the best route if you feel comfortable compiling software from source. It is also the case that many Linux distributions contain very old packages; so unless you�re on a very up-to-date distro or are using backports, installing from source may be the best bet. + +To install Git, you need to have the following libraries that Git depends on: curl, zlib, openssl, expat, and libiconv. For example, if you�re on a system that has yum (such as Fedora) or apt-get (such as a Debian based system), you can use one of these commands to install all of the dependencies: + + $ yum install curl-devel expat-devel gettext-devel \ + openssl-devel zlib-devel + + $ apt-get install libcurl4-gnutls-dev libexpat1-dev gettext \ + libz-dev libssl-dev + +When you have all the necessary dependencies, you can go ahead and grab the latest snapshot from the Git web site: + + http://git-scm.com/download + +Then, compile and install: + + $ tar -zxf git-1.7.2.2.tar.gz + $ cd git-1.7.2.2 + $ make prefix=/usr/local all + $ sudo make prefix=/usr/local install + +After this is done, you can also get Git via Git itself for updates: + + $ git clone git://git.kernel.org/pub/scm/git/git.git + +### Installing on Linux ### + +If you want to install Git on Linux via a binary installer, you can generally do so through the basic package-management tool that comes with your distribution. If you�re on Fedora, you can use yum: + + $ yum install git-core + +Or if you�re on a Debian-based distribution like Ubuntu, try apt-get: + + $ apt-get install git + +### Installing on Mac ### + +There are two easy ways to install Git on a Mac. The easiest is to use the graphical Git installer, which you can download from the Google Code page (see Figure 1-7): + + http://code.google.com/p/git-osx-installer + +Insert 18333fig0107.png +Figure 1-7. Git OS X installer. + +The other major way is to install Git via MacPorts (`http://www.macports.org`). If you have MacPorts installed, install Git via + + $ sudo port install git-core +svn +doc +bash_completion +gitweb + +You don�t have to add all the extras, but you�ll probably want to include +svn in case you ever have to use Git with Subversion repositories (see Chapter 8). + +### Installing on Windows ### + +Installing Git on Windows is very easy. The msysGit project has one of the easier installation procedures. Simply download the installer exe file from the GitHub page, and run it: + + http://msysgit.github.com/ + +After it�s installed, you have both a command-line version (including an SSH client that will come in handy later) and the standard GUI. + +Note on Windows usage: you should use Git with the provided msysGit shell (Unix style), it allows to use the complex lines of command given in this book. If you need, for some reason, to use the native Windows shell / command line console, you have to use double quotes instead of simple quotes (for parameters with spaces in them) and you must quote the parameters ending with the circumflex accent (^) if they are last on the line, as it is a continuation symbol in Windows. + +## First-Time Git Setup ## + +Now that you have Git on your system, you�ll want to do a few things to customize your Git environment. You should have to do these things only once; they�ll stick around between upgrades. You can also change them at any time by running through the commands again. + +Git comes with a tool called git config that lets you get and set configuration variables that control all aspects of how Git looks and operates. These variables can be stored in three different places: + +* `/etc/gitconfig` file: Contains values for every user on the system and all their repositories. If you pass the option` --system` to `git config`, it reads and writes from this file specifically. +* `~/.gitconfig` file: Specific to your user. You can make Git read and write to this file specifically by passing the `--global` option. +* config file in the git directory (that is, `.git/config`) of whatever repository you�re currently using: Specific to that single repository. Each level overrides values in the previous level, so values in `.git/config` trump those in `/etc/gitconfig`. + +On Windows systems, Git looks for the `.gitconfig` file in the `$HOME` directory (`%USERPROFILE%` in Windows� environment), which is `C:\Documents and Settings\$USER` or `C:\Users\$USER` for most people, depending on version (`$USER` is `%USERNAME%` in Windows� environment). It also still looks for /etc/gitconfig, although it�s relative to the MSys root, which is wherever you decide to install Git on your Windows system when you run the installer. + +### Your Identity ### + +The first thing you should do when you install Git is to set your user name and e-mail address. This is important because every Git commit uses this information, and it�s immutably baked into the commits you pass around: + + $ git config --global user.name "John Doe" + $ git config --global user.email [email protected] + +Again, you need to do this only once if you pass the `--global` option, because then Git will always use that information for anything you do on that system. If you want to override this with a different name or e-mail address for specific projects, you can run the command without the `--global` option when you�re in that project. + +### Your Editor ### + +Now that your identity is set up, you can configure the default text editor that will be used when Git needs you to type in a message. By default, Git uses your system�s default editor, which is generally Vi or Vim. If you want to use a different text editor, such as Emacs, you can do the following: + + $ git config --global core.editor emacs + +### Your Diff Tool ### + +Another useful option you may want to configure is the default diff tool to use to resolve merge conflicts. Say you want to use vimdiff: + + $ git config --global merge.tool vimdiff + +Git accepts kdiff3, tkdiff, meld, xxdiff, emerge, vimdiff, gvimdiff, ecmerge, and opendiff as valid merge tools. You can also set up a custom tool; see Chapter 7 for more information about doing that. + +### Checking Your Settings ### + +If you want to check your settings, you can use the `git config --list` command to list all the settings Git can find at that point: + + $ git config --list + user.name=Scott Chacon + color.status=auto + color.branch=auto + color.interactive=auto + color.diff=auto + ... + +You may see keys more than once, because Git reads the same key from different files (`/etc/gitconfig` and `~/.gitconfig`, for example). In this case, Git uses the last value for each unique key it sees. + +You can also check what Git thinks a specific key�s value is by typing `git config {key}`: + + $ git config user.name + Scott Chacon + +## Getting Help ## + +If you ever need help while using Git, there are three ways to get the manual page (manpage) help for any of the Git commands: + + $ git help <verb> + $ git <verb> --help + $ man git-<verb> + +For example, you can get the manpage help for the config command by running + + $ git help config + +These commands are nice because you can access them anywhere, even offline. +If the manpages and this book aren�t enough and you need in-person help, you can try the `#git` or `#github` channel on the Freenode IRC server (irc.freenode.net). These channels are regularly filled with hundreds of people who are all very knowledgeable about Git and are often willing to help. + +## Summary ## + +You should have a basic understanding of what Git is and how it�s different from the CVCS you may have been using. You should also now have a working version of Git on your system that�s set up with your personal identity. It�s now time to learn some Git basics. + diff --git a/resource/bundler/bundler.go b/resource/bundler/bundler.go new file mode 100644 index 000000000..2f3981485 --- /dev/null +++ b/resource/bundler/bundler.go @@ -0,0 +1,121 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bundler contains functions for concatenation etc. of Resource objects. +package bundler + +import ( + "errors" + "fmt" + "io" + "path/filepath" + + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resource" +) + +// Client contains methods perform concatenation and other bundling related +// tasks to Resource objects. +type Client struct { + rs *resource.Spec +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec) *Client { + return &Client{rs: rs} +} + +type multiReadSeekCloser struct { + mr io.Reader + sources []resource.ReadSeekCloser +} + +func (r *multiReadSeekCloser) Read(p []byte) (n int, err error) { + return r.mr.Read(p) +} + +func (r *multiReadSeekCloser) Seek(offset int64, whence int) (newOffset int64, err error) { + for _, s := range r.sources { + newOffset, err = s.Seek(offset, whence) + if err != nil { + return + } + } + return +} + +func (r *multiReadSeekCloser) Close() error { + for _, s := range r.sources { + s.Close() + } + return nil +} + +// Concat concatenates the list of Resource objects. +func (c *Client) Concat(targetPath string, resources []resource.Resource) (resource.Resource, error) { + // The CACHE_OTHER will make sure this will be re-created and published on rebuilds. + return c.rs.ResourceCache.GetOrCreate(resource.CACHE_OTHER, targetPath, func() (resource.Resource, error) { + var resolvedm media.Type + + // The given set of resources must be of the same Media Type. + // We may improve on that in the future, but then we need to know more. + for i, r := range resources { + if i > 0 && r.MediaType() != resolvedm { + return nil, errors.New("resources in Concat must be of the same Media Type") + } + resolvedm = r.MediaType() + } + + concatr := func() (resource.ReadSeekCloser, error) { + var rcsources []resource.ReadSeekCloser + for _, s := range resources { + rcr, ok := s.(resource.ReadSeekCloserResource) + if !ok { + return nil, fmt.Errorf("resource %T does not implement resource.ReadSeekerCloserResource", s) + } + rc, err := rcr.ReadSeekCloser() + if err != nil { + // Close the already opened. + for _, rcs := range rcsources { + rcs.Close() + } + return nil, err + } + rcsources = append(rcsources, rc) + } + + readers := make([]io.Reader, len(rcsources)) + for i := 0; i < len(rcsources); i++ { + readers[i] = rcsources[i] + } + + mr := io.MultiReader(readers...) + + return &multiReadSeekCloser{mr: mr, sources: rcsources}, nil + } + + composite, err := c.rs.NewForFs( + c.rs.BaseFs.Resources.Fs, + resource.ResourceSourceDescriptor{ + LazyPublish: true, + OpenReadSeekCloser: concatr, + RelTargetFilename: filepath.Clean(targetPath)}) + + if err != nil { + return nil, err + } + + return composite, nil + }) + +} diff --git a/resource/create/create.go b/resource/create/create.go new file mode 100644 index 000000000..1c7894232 --- /dev/null +++ b/resource/create/create.go @@ -0,0 +1,77 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package create contains functions for to create Resource objects. This will +// typically non-files. +package create + +import ( + "io" + "path/filepath" + + "github.com/spf13/afero" + + "github.com/dsnet/golib/memfile" + "github.com/gohugoio/hugo/resource" +) + +// Client contains methods to create Resource objects. +// tasks to Resource objects. +type Client struct { + rs *resource.Spec +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec) *Client { + return &Client{rs: rs} +} + +type memFileCloser struct { + *memfile.File + io.Closer +} + +func (m *memFileCloser) Close() error { + return nil +} + +// Get creates a new Resource by opening the given filename in the given filesystem. +func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) { + filename = filepath.Clean(filename) + return c.rs.ResourceCache.GetOrCreate(resource.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) { + return c.rs.NewForFs(fs, + resource.ResourceSourceDescriptor{ + LazyPublish: true, + SourceFilename: filename}) + + }) + +} + +// FromString creates a new Resource from a string with the given relative target path. +func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(resource.CACHE_OTHER, targetPath, func() (resource.Resource, error) { + return c.rs.NewForFs( + c.rs.BaseFs.Resources.Fs, + resource.ResourceSourceDescriptor{ + LazyPublish: true, + OpenReadSeekCloser: func() (resource.ReadSeekCloser, error) { + return &memFileCloser{ + File: memfile.New([]byte(content)), + }, nil + }, + RelTargetFilename: filepath.Clean(targetPath)}) + + }) + +} diff --git a/resource/image.go b/resource/image.go index 19b68a296..6aa382331 100644 --- a/resource/image.go +++ b/resource/image.go @@ -19,14 +19,12 @@ import ( "image/color" "io" "os" - "path/filepath" "strconv" "strings" "github.com/mitchellh/mapstructure" "github.com/gohugoio/hugo/helpers" - "github.com/spf13/afero" // Importing image codecs for image.DecodeConfig "image" @@ -132,8 +130,6 @@ type Image struct { format imaging.Format - hash string - *genericResource } @@ -151,7 +147,6 @@ func (i *Image) Height() int { func (i *Image) WithNewBase(base string) Resource { return &Image{ imaging: i.imaging, - hash: i.hash, format: i.format, genericResource: i.genericResource.WithNewBase(base).(*genericResource)} } @@ -209,7 +204,7 @@ type imageConfig struct { } func (i *Image) isJPEG() bool { - name := strings.ToLower(i.relTargetPath.file) + name := strings.ToLower(i.relTargetDirFile.file) return strings.HasSuffix(name, ".jpg") || strings.HasSuffix(name, ".jpeg") } @@ -241,7 +236,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c ci := i.clone() errOp := action - errPath := i.AbsSourceFilename() + errPath := i.sourceFilename ci.setBasePath(conf) @@ -273,7 +268,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c ci.config = image.Config{Width: b.Max.X, Height: b.Max.Y} ci.configLoaded = true - return ci, i.encodeToDestinations(converted, conf, resourceCacheFilename, ci.target()) + return ci, i.encodeToDestinations(converted, conf, resourceCacheFilename, ci.targetFilename()) }) } @@ -415,11 +410,11 @@ func (i *Image) initConfig() error { } var ( - f afero.File + f ReadSeekCloser config image.Config ) - f, err = i.sourceFs().Open(i.AbsSourceFilename()) + f, err = i.ReadSeekCloser() if err != nil { return } @@ -440,19 +435,19 @@ func (i *Image) initConfig() error { } func (i *Image) decodeSource() (image.Image, error) { - file, err := i.sourceFs().Open(i.AbsSourceFilename()) + f, err := i.ReadSeekCloser() if err != nil { return nil, fmt.Errorf("failed to open image for decode: %s", err) } - defer file.Close() - img, _, err := image.Decode(file) + defer f.Close() + img, _, err := image.Decode(f) return img, err } func (i *Image) copyToDestination(src string) error { var res error i.copyToDestinationInit.Do(func() { - target := i.target() + target := i.targetFilename() // Fast path: // This is a processed version of the original. @@ -469,20 +464,9 @@ func (i *Image) copyToDestination(src string) error { } defer in.Close() - out, err := i.spec.BaseFs.PublishFs.Create(target) - if err != nil && os.IsNotExist(err) { - // When called from shortcodes, the target directory may not exist yet. - // See https://github.com/gohugoio/hugo/issues/4202 - if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil { - res = err - return - } - out, err = i.spec.BaseFs.PublishFs.Create(target) - if err != nil { - res = err - return - } - } else if err != nil { + out, err := openFileForWriting(i.spec.BaseFs.PublishFs, target) + + if err != nil { res = err return } @@ -501,21 +485,10 @@ func (i *Image) copyToDestination(src string) error { return nil } -func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename, filename string) error { - target := filepath.Clean(filename) +func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename, targetFilename string) error { - file1, err := i.spec.BaseFs.PublishFs.Create(target) - if err != nil && os.IsNotExist(err) { - // When called from shortcodes, the target directory may not exist yet. - // See https://github.com/gohugoio/hugo/issues/4202 - if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil { - return err - } - file1, err = i.spec.BaseFs.PublishFs.Create(target) - if err != nil { - return err - } - } else if err != nil { + file1, err := openFileForWriting(i.spec.BaseFs.PublishFs, targetFilename) + if err != nil { return err } @@ -525,11 +498,7 @@ func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resource if resourceCacheFilename != "" { // Also save it to the image resource cache for later reuse. - if err = i.spec.BaseFs.ResourcesFs.MkdirAll(filepath.Dir(resourceCacheFilename), os.FileMode(0755)); err != nil { - return err - } - - file2, err := i.spec.BaseFs.ResourcesFs.Create(resourceCacheFilename) + file2, err := openFileForWriting(i.spec.BaseFs.Resources.Fs, resourceCacheFilename) if err != nil { return err } @@ -572,17 +541,16 @@ func (i *Image) clone() *Image { return &Image{ imaging: i.imaging, - hash: i.hash, format: i.format, genericResource: &g} } func (i *Image) setBasePath(conf imageConfig) { - i.relTargetPath = i.relTargetPathFromConfig(conf) + i.relTargetDirFile = i.relTargetPathFromConfig(conf) } func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile { - p1, p2 := helpers.FileAndExt(i.relTargetPath.file) + p1, p2 := helpers.FileAndExt(i.relTargetDirFile.file) idStr := fmt.Sprintf("_hu%s_%d", i.hash, i.osFileInfo.Size()) @@ -611,7 +579,7 @@ func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile { } return dirFile{ - dir: i.relTargetPath.dir, + dir: i.relTargetDirFile.dir, file: fmt.Sprintf("%s%s_%s%s", p1, idStr, key, p2), } diff --git a/resource/image_cache.go b/resource/image_cache.go index 5985797d6..4fb45c17f 100644 --- a/resource/image_cache.go +++ b/resource/image_cache.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -60,12 +60,6 @@ func (c *imageCache) getOrCreate( relTarget := parent.relTargetPathFromConfig(conf) key := parent.relTargetPathForRel(relTarget.path(), false) - if c.pathSpec.Language != nil { - // Avoid do and store more work than needed. The language versions will in - // most cases be duplicates of the same image files. - key = strings.TrimPrefix(key, "/"+c.pathSpec.Language.Lang) - } - // First check the in-memory store, then the disk. c.mu.RLock() img, found := c.store[key] @@ -88,17 +82,17 @@ func (c *imageCache) getOrCreate( // but the count of processed image variations for this site. c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) - exists, err := helpers.Exists(cacheFilename, c.pathSpec.BaseFs.ResourcesFs) + exists, err := helpers.Exists(cacheFilename, c.pathSpec.BaseFs.Resources.Fs) if err != nil { return nil, err } if exists { img = parent.clone() - img.relTargetPath.file = relTarget.file + img.relTargetDirFile.file = relTarget.file img.sourceFilename = cacheFilename - // We have to look resources file system for this. - img.overriddenSourceFs = img.spec.BaseFs.ResourcesFs + // We have to look in the resources file system for this. + img.overriddenSourceFs = img.spec.BaseFs.Resources.Fs } else { img, err = create(cacheFilename) if err != nil { diff --git a/resource/image_test.go b/resource/image_test.go index 11807d695..f4d91bd99 100644 --- a/resource/image_test.go +++ b/resource/image_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -78,19 +78,19 @@ func TestImageTransformBasic(t *testing.T) { assert.NoError(err) assert.Equal(320, resized0x.Width()) assert.Equal(200, resized0x.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized0x.RelPermalink(), 320, 200) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resized0x.RelPermalink(), 320, 200) resizedx0, err := image.Resize("200x") assert.NoError(err) assert.Equal(200, resizedx0.Width()) assert.Equal(125, resizedx0.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedx0.RelPermalink(), 200, 125) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedx0.RelPermalink(), 200, 125) resizedAndRotated, err := image.Resize("x200 r90") assert.NoError(err) assert.Equal(125, resizedAndRotated.Width()) assert.Equal(200, resizedAndRotated.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAndRotated.RelPermalink(), 125, 200) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedAndRotated.RelPermalink(), 125, 200) assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_300x200_resize_q68_linear.jpg", resized.RelPermalink()) assert.Equal(300, resized.Width()) @@ -115,20 +115,20 @@ func TestImageTransformBasic(t *testing.T) { assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_bottomleft.jpg", filled.RelPermalink()) assert.Equal(200, filled.Width()) assert.Equal(100, filled.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filled.RelPermalink(), 200, 100) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, filled.RelPermalink(), 200, 100) smart, err := image.Fill("200x100 smart") assert.NoError(err) assert.Equal(fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", smartCropVersionNumber), smart.RelPermalink()) assert.Equal(200, smart.Width()) assert.Equal(100, smart.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, smart.RelPermalink(), 200, 100) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, smart.RelPermalink(), 200, 100) // Check cache filledAgain, err := image.Fill("200x100 bottomLeft") assert.NoError(err) assert.True(filled == filledAgain) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filledAgain.RelPermalink(), 200, 100) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, filledAgain.RelPermalink(), 200, 100) } @@ -298,7 +298,7 @@ func TestImageResizeInSubPath(t *testing.T) { assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resized.RelPermalink()) assert.Equal(101, resized.Width()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized.RelPermalink(), 101, 101) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resized.RelPermalink(), 101, 101) publishedImageFilename := filepath.Clean(resized.RelPermalink()) assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) assert.NoError(image.spec.BaseFs.PublishFs.Remove(publishedImageFilename)) @@ -310,7 +310,7 @@ func TestImageResizeInSubPath(t *testing.T) { assert.NoError(err) assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resizedAgain.RelPermalink()) assert.Equal(101, resizedAgain.Width()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAgain.RelPermalink(), 101, 101) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedAgain.RelPermalink(), 101, 101) assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) } diff --git a/resource/integrity/integrity.go b/resource/integrity/integrity.go new file mode 100644 index 000000000..8b4a5a263 --- /dev/null +++ b/resource/integrity/integrity.go @@ -0,0 +1,106 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package integrity + +import ( + "crypto/md5" + "crypto/sha256" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "fmt" + "hash" + "io" + + "github.com/gohugoio/hugo/resource" +) + +const defaultHashAlgo = "sha256" + +// Client contains methods to fingerprint (cachebusting) and other integrity-related +// methods. +type Client struct { + rs *resource.Spec +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec) *Client { + return &Client{rs: rs} +} + +type fingerprintTransformation struct { + algo string +} + +func (t *fingerprintTransformation) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("fingerprint", t.algo) +} + +// Transform creates a MD5 hash of the Resource content and inserts that hash before +// the extension in the filename. +func (t *fingerprintTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + algo := t.algo + + var h hash.Hash + + switch algo { + case "md5": + h = md5.New() + case "sha256": + h = sha256.New() + case "sha512": + h = sha512.New() + default: + return fmt.Errorf("unsupported crypto algo: %q, use either md5, sha256 or sha512", algo) + } + + io.Copy(io.MultiWriter(h, ctx.To), ctx.From) + d, err := digest(h) + if err != nil { + return err + } + + ctx.Data["Integrity"] = integrity(algo, d) + ctx.AddOutPathIdentifier("." + hex.EncodeToString(d[:])) + return nil +} + +// Fingerprint applies fingerprinting of the given resource and hash algorithm. +// It defaults to sha256 if none given, and the options are md5, sha256 or sha512. +// The same algo is used for both the fingerprinting part (aka cache busting) and +// the base64-encoded Subresource Integrity hash, so you will have to stay away from +// md5 if you plan to use both. +// See https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity +func (c *Client) Fingerprint(res resource.Resource, algo string) (resource.Resource, error) { + if algo == "" { + algo = defaultHashAlgo + } + + return c.rs.Transform( + res, + &fingerprintTransformation{algo: algo}, + ) +} + +func integrity(algo string, sum []byte) string { + encoded := base64.StdEncoding.EncodeToString(sum) + return fmt.Sprintf("%s-%s", algo, encoded) + +} + +func digest(h hash.Hash) ([]byte, error) { + sum := h.Sum(nil) + //enc := hex.EncodeToString(sum[:]) + return sum, nil +} diff --git a/resource/integrity/integrity_test.go b/resource/integrity/integrity_test.go new file mode 100644 index 000000000..602db4e38 --- /dev/null +++ b/resource/integrity/integrity_test.go @@ -0,0 +1,54 @@ +// Copyright 2018-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package integrity + +import ( + "github.com/gohugoio/hugo/media" +) + +type testResource struct { + content string +} + +func (r testResource) Permalink() string { + panic("not implemented") +} + +func (r testResource) RelPermalink() string { + panic("not implemented") +} + +func (r testResource) ResourceType() string { + panic("not implemented") +} + +func (r testResource) Name() string { + panic("not implemented") +} + +func (r testResource) MediaType() media.Type { + panic("not implemented") +} + +func (r testResource) Title() string { + panic("not implemented") +} + +func (r testResource) Params() map[string]interface{} { + panic("not implemented") +} + +func (r testResource) Bytes() ([]byte, error) { + return []byte(r.content), nil +} diff --git a/resource/minifiers/minify.go b/resource/minifiers/minify.go new file mode 100644 index 000000000..609b9a694 --- /dev/null +++ b/resource/minifiers/minify.go @@ -0,0 +1,115 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package minifiers + +import ( + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/media" + + "github.com/gohugoio/hugo/resource" + "github.com/tdewolff/minify" + "github.com/tdewolff/minify/css" + "github.com/tdewolff/minify/html" + "github.com/tdewolff/minify/js" + "github.com/tdewolff/minify/json" + "github.com/tdewolff/minify/svg" + "github.com/tdewolff/minify/xml" +) + +// Client for minification of Resource objects. Supported minfiers are: +// css, html, js, json, svg and xml. +type Client struct { + rs *resource.Spec + m *minify.M +} + +// New creates a new Client given a specification. Note that it is the media types +// configured for the site that is used to match files to the correct minifier. +func New(rs *resource.Spec) *Client { + m := minify.New() + mt := rs.MediaTypes + + // We use the Type definition of the media types defined in the site if found. + addMinifierFunc(m, mt, "text/css", "css", css.Minify) + addMinifierFunc(m, mt, "text/html", "html", html.Minify) + addMinifierFunc(m, mt, "application/javascript", "js", js.Minify) + addMinifierFunc(m, mt, "application/json", "json", json.Minify) + addMinifierFunc(m, mt, "image/svg", "xml", svg.Minify) + addMinifierFunc(m, mt, "application/xml", "xml", xml.Minify) + addMinifierFunc(m, mt, "application/rss", "xml", xml.Minify) + + return &Client{rs: rs, m: m} +} + +func addMinifierFunc(m *minify.M, mt media.Types, typeString, suffix string, fn minify.MinifierFunc) { + resolvedTypeStr := resolveMediaTypeString(mt, typeString, suffix) + m.AddFunc(resolvedTypeStr, fn) + if resolvedTypeStr != typeString { + m.AddFunc(typeString, fn) + } +} + +type minifyTransformation struct { + rs *resource.Spec + m *minify.M +} + +func (t *minifyTransformation) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("minify") +} + +func (t *minifyTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + mtype := resolveMediaTypeString( + t.rs.MediaTypes, + ctx.InMediaType.Type(), + helpers.ExtNoDelimiter(ctx.InPath), + ) + if err := t.m.Minify(mtype, ctx.To, ctx.From); err != nil { + return err + } + ctx.AddOutPathIdentifier(".min") + return nil +} + +func (c *Client) Minify(res resource.Resource) (resource.Resource, error) { + return c.rs.Transform( + res, + &minifyTransformation{ + rs: c.rs, + m: c.m}, + ) +} + +func resolveMediaTypeString(types media.Types, typeStr, suffix string) string { + if m, found := resolveMediaType(types, typeStr, suffix); found { + return m.Type() + } + // Fall back to the default. + return typeStr +} + +// Make sure we match the matching pattern with what the user have actually defined +// in his or hers media types configuration. +func resolveMediaType(types media.Types, typeStr, suffix string) (media.Type, bool) { + if m, found := types.GetByType(typeStr); found { + return m, true + } + + if m, found := types.GetFirstBySuffix(suffix); found { + return m, true + } + + return media.Type{}, false + +} diff --git a/resource/postcss/postcss.go b/resource/postcss/postcss.go new file mode 100644 index 000000000..7dd27b2f9 --- /dev/null +++ b/resource/postcss/postcss.go @@ -0,0 +1,175 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postcss + +import ( + "fmt" + "io" + "path/filepath" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/mitchellh/mapstructure" + // "io/ioutil" + "os" + "os/exec" + + "github.com/gohugoio/hugo/common/errors" + + "github.com/gohugoio/hugo/resource" +) + +// Some of the options from https://github.com/postcss/postcss-cli +type Options struct { + + // Set a custom path to look for a config file. + Config string + + NoMap bool `mapstructure:"no-map"` // Disable the default inline sourcemaps + + // Options for when not using a config file + Use string // List of postcss plugins to use + Parser string // Custom postcss parser + Stringifier string // Custom postcss stringifier + Syntax string // Custom postcss syntax +} + +func DecodeOptions(m map[string]interface{}) (opts Options, err error) { + if m == nil { + return + } + err = mapstructure.WeakDecode(m, &opts) + return +} + +func (opts Options) toArgs() []string { + var args []string + if opts.NoMap { + args = append(args, "--no-map") + } + if opts.Use != "" { + args = append(args, "--use", opts.Use) + } + if opts.Parser != "" { + args = append(args, "--parser", opts.Parser) + } + if opts.Stringifier != "" { + args = append(args, "--stringifier", opts.Stringifier) + } + if opts.Syntax != "" { + args = append(args, "--syntax", opts.Syntax) + } + return args +} + +// Client is the client used to do PostCSS transformations. +type Client struct { + rs *resource.Spec +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec) *Client { + return &Client{rs: rs} +} + +type postcssTransformation struct { + options Options + rs *resource.Spec +} + +func (t *postcssTransformation) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("postcss", t.options) +} + +// Transform shells out to postcss-cli to do the heavy lifting. +// For this to work, you need some additional tools. To install them globally: +// npm install -g postcss-cli +// npm install -g autoprefixer +func (t *postcssTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + + const binary = "postcss" + + if _, err := exec.LookPath(binary); err != nil { + // This may be on a CI server etc. Will fall back to pre-built assets. + return errors.FeatureNotAvailableErr + } + + var configFile string + logger := t.rs.Logger + + if t.options.Config != "" { + configFile = t.options.Config + } else { + configFile = "postcss.config.js" + } + + configFile = filepath.Clean(configFile) + + // We need an abolute filename to the config file. + if !filepath.IsAbs(configFile) { + // We resolve this against the virtual Work filesystem, to allow + // this config file to live in one of the themes if needed. + fi, err := t.rs.BaseFs.Work.Fs.Stat(configFile) + if err != nil { + if t.options.Config != "" { + // Only fail if the user specificed config file is not found. + return fmt.Errorf("postcss config %q not found: %s", configFile, err) + } + configFile = "" + } else { + configFile = fi.(hugofs.RealFilenameInfo).RealFilename() + } + } + + var cmdArgs []string + + if configFile != "" { + logger.INFO.Println("postcss: use config file", configFile) + cmdArgs = []string{"--config", configFile} + } + + if optArgs := t.options.toArgs(); len(optArgs) > 0 { + cmdArgs = append(cmdArgs, optArgs...) + } + + cmd := exec.Command(binary, cmdArgs...) + + cmd.Stdout = ctx.To + cmd.Stderr = os.Stderr + + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + + go func() { + defer stdin.Close() + io.Copy(stdin, ctx.From) + }() + + err = cmd.Run() + if err != nil { + return err + } + + return nil +} + +// Process transforms the given Resource with the PostCSS processor. +func (c *Client) Process(res resource.Resource, options Options) (resource.Resource, error) { + return c.rs.Transform( + res, + &postcssTransformation{rs: c.rs, options: options}, + ) +} diff --git a/resource/resource.go b/resource/resource.go index 9a3725f8a..f0989e51e 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,20 +14,24 @@ package resource import ( + "errors" "fmt" + "io" + "io/ioutil" "mime" "os" "path" "path/filepath" - "strconv" "strings" "sync" - "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/tpl" - "github.com/spf13/afero" + "github.com/gohugoio/hugo/common/loggers" + + jww "github.com/spf13/jwalterweatherman" - "github.com/spf13/cast" + "github.com/spf13/afero" "github.com/gobwas/glob" "github.com/gohugoio/hugo/helpers" @@ -36,34 +40,39 @@ import ( ) var ( + _ ContentResource = (*genericResource)(nil) + _ ReadSeekCloserResource = (*genericResource)(nil) _ Resource = (*genericResource)(nil) - _ metaAssigner = (*genericResource)(nil) _ Source = (*genericResource)(nil) _ Cloner = (*genericResource)(nil) _ ResourcesLanguageMerger = (*Resources)(nil) + _ permalinker = (*genericResource)(nil) ) const DefaultResourceType = "unknown" +var noData = make(map[string]interface{}) + // Source is an internal template and not meant for use in the templates. It // may change without notice. type Source interface { - AbsSourceFilename() string Publish() error } +type permalinker interface { + relPermalinkFor(target string) string + permalinkFor(target string) string + relTargetPathFor(target string) string + relTargetPath() string + targetPath() string +} + // Cloner is an internal template and not meant for use in the templates. It // may change without notice. type Cloner interface { WithNewBase(base string) Resource } -type metaAssigner interface { - setTitle(title string) - setName(name string) - updateParams(params map[string]interface{}) -} - // Resource represents a linkable resource, i.e. a content page, image etc. type Resource interface { // Permalink represents the absolute link to this resource. @@ -77,6 +86,9 @@ type Resource interface { // For content pages, this value is "page". ResourceType() string + // MediaType is this resource's MIME type. + MediaType() media.Type + // Name is the logical name of this resource. This can be set in the front matter // metadata for this resource. If not set, Hugo will assign a value. // This will in most cases be the base filename. @@ -88,8 +100,30 @@ type Resource interface { // Title returns the title if set in front matter. For content pages, this will be the expected value. Title() string + // Resource specific data set by Hugo. + // One example would be.Data.Digest for fingerprinted resources. + Data() interface{} + // Params set in front matter for this resource. Params() map[string]interface{} +} + +type ResourcesLanguageMerger interface { + MergeByLanguage(other Resources) Resources + // Needed for integration with the tpl package. + MergeByLanguageInterface(other interface{}) (interface{}, error) +} + +type translatedResource interface { + TranslationKey() string +} + +// ContentResource represents a Resource that provides a way to get to its content. +// Most Resource types in Hugo implements this interface, including Page. +// This should be used with care, as it will read the file content into memory, but it +// should be cached as effectively as possible by the implementation. +type ContentResource interface { + Resource // Content returns this resource's content. It will be equivalent to reading the content // that RelPermalink points to in the published folder. @@ -100,14 +134,22 @@ type Resource interface { Content() (interface{}, error) } -type ResourcesLanguageMerger interface { - MergeByLanguage(other Resources) Resources - // Needed for integration with the tpl package. - MergeByLanguageInterface(other interface{}) (interface{}, error) +// ReadSeekCloser is implemented by afero.File. We use this as the common type for +// content in Resource objects, even for strings. +type ReadSeekCloser interface { + io.Reader + io.Seeker + io.Closer } -type translatedResource interface { - TranslationKey() string +// OpenReadSeekeCloser allows setting some other way (than reading from a filesystem) +// to open or create a ReadSeekCloser. +type OpenReadSeekCloser func() (ReadSeekCloser, error) + +// ReadSeekCloserResource is a Resource that supports loading its content. +type ReadSeekCloserResource interface { + Resource + ReadSeekCloser() (ReadSeekCloser, error) } // Resources represents a slice of resources, which can be a mix of different types. @@ -125,44 +167,6 @@ func (r Resources) ByType(tp string) Resources { return filtered } -const prefixDeprecatedMsg = `We have added the more flexible Resources.GetMatch (find one) and Resources.Match (many) to replace the "prefix" methods. - -These matches by a given globbing pattern, e.g. "*.jpg". - -Some examples: - -* To find all resources by its prefix in the root dir of the bundle: .Match image* -* To find one resource by its prefix in the root dir of the bundle: .GetMatch image* -* To find all JPEG images anywhere in the bundle: .Match **.jpg` - -// GetByPrefix gets the first resource matching the given filename prefix, e.g -// "logo" will match logo.png. It returns nil of none found. -// In potential ambiguous situations, combine it with ByType. -func (r Resources) GetByPrefix(prefix string) Resource { - helpers.Deprecated("Resources", "GetByPrefix", prefixDeprecatedMsg, true) - prefix = strings.ToLower(prefix) - for _, resource := range r { - if matchesPrefix(resource, prefix) { - return resource - } - } - return nil -} - -// ByPrefix gets all resources matching the given base filename prefix, e.g -// "logo" will match logo.png. -func (r Resources) ByPrefix(prefix string) Resources { - helpers.Deprecated("Resources", "ByPrefix", prefixDeprecatedMsg, true) - var matches Resources - prefix = strings.ToLower(prefix) - for _, resource := range r { - if matchesPrefix(resource, prefix) { - matches = append(matches, resource) - } - } - return matches -} - // GetMatch finds the first Resource matching the given pattern, or nil if none found. // See Match for a more complete explanation about the rules used. func (r Resources) GetMatch(pattern string) Resource { @@ -204,10 +208,6 @@ func (r Resources) Match(pattern string) Resources { return matches } -func matchesPrefix(r Resource, prefix string) bool { - return strings.HasPrefix(strings.ToLower(r.Name()), prefix) -} - var ( globCache = make(map[string]glob.Glob) globMu sync.RWMutex @@ -268,81 +268,180 @@ func (r1 Resources) MergeByLanguageInterface(in interface{}) (interface{}, error type Spec struct { *helpers.PathSpec - mimeTypes media.Types + MediaTypes media.Types + + Logger *jww.Notepad + + TextTemplates tpl.TemplateParseFinder // Holds default filter settings etc. imaging *Imaging - imageCache *imageCache + imageCache *imageCache + ResourceCache *ResourceCache - GenImagePath string + GenImagePath string + GenAssetsPath string } -func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) { +func NewSpec(s *helpers.PathSpec, logger *jww.Notepad, mimeTypes media.Types) (*Spec, error) { imaging, err := decodeImaging(s.Cfg.GetStringMap("imaging")) if err != nil { return nil, err } - genImagePath := filepath.FromSlash("_gen/images") + if logger == nil { + logger = loggers.NewErrorLogger() + } - return &Spec{PathSpec: s, - GenImagePath: genImagePath, - imaging: &imaging, mimeTypes: mimeTypes, imageCache: newImageCache( + genImagePath := filepath.FromSlash("_gen/images") + // The transformed assets (CSS etc.) + genAssetsPath := filepath.FromSlash("_gen/assets") + + rs := &Spec{PathSpec: s, + Logger: logger, + GenImagePath: genImagePath, + GenAssetsPath: genAssetsPath, + imaging: &imaging, + MediaTypes: mimeTypes, + imageCache: newImageCache( s, // We're going to write a cache pruning routine later, so make it extremely // unlikely that the user shoots him or herself in the foot // and this is set to a value that represents data he/she // cares about. This should be set in stone once released. genImagePath, - )}, nil -} + )} -func (r *Spec) NewResourceFromFile( - targetPathBuilder func(base string) string, - file source.File, relTargetFilename string) (Resource, error) { + rs.ResourceCache = newResourceCache(rs) + + return rs, nil - return r.newResource(targetPathBuilder, file.Filename(), file.FileInfo(), relTargetFilename) } -func (r *Spec) NewResourceFromFilename( - targetPathBuilder func(base string) string, - absSourceFilename, relTargetFilename string) (Resource, error) { +type ResourceSourceDescriptor struct { + // TargetPathBuilder is a callback to create target paths's relative to its owner. + TargetPathBuilder func(base string) string - fi, err := r.sourceFs().Stat(absSourceFilename) - if err != nil { - return nil, err + // Need one of these to load the resource content. + SourceFile source.File + OpenReadSeekCloser OpenReadSeekCloser + + // If OpenReadSeekerCloser is not set, we use this to open the file. + SourceFilename string + + // The relative target filename without any language code. + RelTargetFilename string + + // Any base path prepeneded to the permalink. + // Typically the language code if this resource should be published to its sub-folder. + URLBase string + + // Any base path prepended to the target path. This will also typically be the + // language code, but setting it here means that it should not have any effect on + // the permalink. + TargetPathBase string + + // Delay publishing until either Permalink or RelPermalink is called. Maybe never. + LazyPublish bool +} + +func (r ResourceSourceDescriptor) Filename() string { + if r.SourceFile != nil { + return r.SourceFile.Filename() } - return r.newResource(targetPathBuilder, absSourceFilename, fi, relTargetFilename) + return r.SourceFilename } func (r *Spec) sourceFs() afero.Fs { - return r.PathSpec.BaseFs.ContentFs + return r.PathSpec.BaseFs.Content.Fs } -func (r *Spec) newResource( - targetPathBuilder func(base string) string, - absSourceFilename string, fi os.FileInfo, relTargetFilename string) (Resource, error) { +func (r *Spec) New(fd ResourceSourceDescriptor) (Resource, error) { + return r.newResourceForFs(r.sourceFs(), fd) +} - var mimeType string - ext := filepath.Ext(relTargetFilename) - m, found := r.mimeTypes.GetBySuffix(strings.TrimPrefix(ext, ".")) - if found { - mimeType = m.SubType - } else { - mimeType = mime.TypeByExtension(ext) - if mimeType == "" { - mimeType = DefaultResourceType - } else { - mimeType = mimeType[:strings.Index(mimeType, "/")] +func (r *Spec) NewForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Resource, error) { + return r.newResourceForFs(sourceFs, fd) +} + +func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Resource, error) { + if fd.OpenReadSeekCloser == nil { + if fd.SourceFile != nil && fd.SourceFilename != "" { + return nil, errors.New("both SourceFile and AbsSourceFilename provided") + } else if fd.SourceFile == nil && fd.SourceFilename == "" { + return nil, errors.New("either SourceFile or AbsSourceFilename must be provided") } } - gr := r.newGenericResource(targetPathBuilder, fi, absSourceFilename, relTargetFilename, mimeType) + if fd.URLBase == "" { + fd.URLBase = r.GetURLLanguageBasePath() + } + + if fd.TargetPathBase == "" { + fd.TargetPathBase = r.GetTargetLanguageBasePath() + } + + if fd.RelTargetFilename == "" { + fd.RelTargetFilename = fd.Filename() + } + + return r.newResource(sourceFs, fd) +} - if mimeType == "image" { - ext := strings.ToLower(helpers.Ext(absSourceFilename)) +func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Resource, error) { + var fi os.FileInfo + var sourceFilename string + + if fd.OpenReadSeekCloser != nil { + + } else if fd.SourceFilename != "" { + var err error + fi, err = sourceFs.Stat(fd.SourceFilename) + if err != nil { + return nil, err + } + sourceFilename = fd.SourceFilename + } else { + fi = fd.SourceFile.FileInfo() + sourceFilename = fd.SourceFile.Filename() + } + + if fd.RelTargetFilename == "" { + fd.RelTargetFilename = sourceFilename + } + + ext := filepath.Ext(fd.RelTargetFilename) + mimeType, found := r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, ".")) + // TODO(bep) we need to handle these ambigous types better, but in this context + // we most likely want the application/xml type. + if mimeType.Suffix == "xml" && mimeType.SubType == "rss" { + mimeType, found = r.MediaTypes.GetByType("application/xml") + } + + if !found { + mimeStr := mime.TypeByExtension(ext) + if mimeStr != "" { + mimeType, _ = media.FromString(mimeStr) + } + + } + + gr := r.newGenericResourceWithBase( + sourceFs, + fd.LazyPublish, + fd.OpenReadSeekCloser, + fd.URLBase, + fd.TargetPathBase, + fd.TargetPathBuilder, + fi, + sourceFilename, + fd.RelTargetFilename, + mimeType) + + if mimeType.MainType == "image" { + ext := strings.ToLower(helpers.Ext(sourceFilename)) imgFormat, ok := imageFormats[ext] if !ok { @@ -351,27 +450,21 @@ func (r *Spec) newResource( return gr, nil } - f, err := gr.sourceFs().Open(absSourceFilename) - if err != nil { - return nil, fmt.Errorf("failed to open image source file: %s", err) - } - defer f.Close() - - hash, err := helpers.MD5FromFileFast(f) - if err != nil { + if err := gr.initHash(); err != nil { return nil, err } return &Image{ - hash: hash, format: imgFormat, imaging: r.imaging, genericResource: gr}, nil } return gr, nil + } -func (r *Spec) IsInCache(key string) bool { +// TODO(bep) unify +func (r *Spec) IsInImageCache(key string) bool { // This is used for cache pruning. We currently only have images, but we could // imagine expanding on this. return r.imageCache.isInCache(key) @@ -381,6 +474,11 @@ func (r *Spec) DeleteCacheByPrefix(prefix string) { r.imageCache.deleteByPrefix(prefix) } +func (r *Spec) ClearCaches() { + r.imageCache.clear() + r.ResourceCache.clear() +} + func (r *Spec) CacheStats() string { r.imageCache.mu.RLock() defer r.imageCache.mu.RUnlock() @@ -410,18 +508,54 @@ func (d dirFile) path() string { return path.Join(d.dir, d.file) } +type resourcePathDescriptor struct { + // The relative target directory and filename. + relTargetDirFile dirFile + + // Callback used to construct a target path relative to its owner. + targetPathBuilder func(rel string) string + + // baseURLDir is the fixed sub-folder for a resource in permalinks. This will typically + // be the language code if we publish to the language's sub-folder. + baseURLDir string + + // This will normally be the same as above, but this will only apply to publishing + // of resources. + baseTargetPathDir string + + // baseOffset is set when the output format's path has a offset, e.g. for AMP. + baseOffset string +} + type resourceContent struct { content string contentInit sync.Once } +type resourceHash struct { + hash string + hashInit sync.Once +} + +type publishOnce struct { + publisherInit sync.Once + publisherErr error + logger *jww.Notepad +} + +func (l *publishOnce) publish(s Source) error { + l.publisherInit.Do(func() { + l.publisherErr = s.Publish() + if l.publisherErr != nil { + l.logger.ERROR.Printf("failed to publish Resource: %s", l.publisherErr) + } + }) + return l.publisherErr +} + // genericResource represents a generic linkable resource. type genericResource struct { - // The relative path to this resource. - relTargetPath dirFile - - // Base is set when the output format's path has a offset, e.g. for AMP. - base string + resourcePathDescriptor title string name string @@ -433,6 +567,12 @@ type genericResource struct { // the path to the file on the real filesystem. sourceFilename string + // Will be set if this resource is backed by something other than a file. + openReadSeekerCloser OpenReadSeekCloser + + // A hash of the source content. Is only calculated in caching situations. + *resourceHash + // This may be set to tell us to look in another filesystem for this resource. // We, by default, use the sourceFs filesystem in the spec below. overriddenSourceFs afero.Fs @@ -440,20 +580,87 @@ type genericResource struct { spec *Spec resourceType string - osFileInfo os.FileInfo + mediaType media.Type - targetPathBuilder func(rel string) string + osFileInfo os.FileInfo // We create copies of this struct, so this needs to be a pointer. *resourceContent + + // May be set to signal lazy/delayed publishing. + *publishOnce +} + +func (l *genericResource) Data() interface{} { + return noData } func (l *genericResource) Content() (interface{}, error) { + if err := l.initContent(); err != nil { + return nil, err + } + + return l.content, nil +} + +func (l *genericResource) ReadSeekCloser() (ReadSeekCloser, error) { + if l.openReadSeekerCloser != nil { + return l.openReadSeekerCloser() + } + f, err := l.sourceFs().Open(l.sourceFilename) + if err != nil { + return nil, err + } + return f, nil + +} + +func (l *genericResource) MediaType() media.Type { + return l.mediaType +} + +// Implement the Cloner interface. +func (l genericResource) WithNewBase(base string) Resource { + l.baseOffset = base + l.resourceContent = &resourceContent{} + return &l +} + +func (l *genericResource) initHash() error { + var err error + l.hashInit.Do(func() { + var hash string + var f ReadSeekCloser + f, err = l.ReadSeekCloser() + if err != nil { + err = fmt.Errorf("failed to open source file: %s", err) + return + } + defer f.Close() + + hash, err = helpers.MD5FromFileFast(f) + if err != nil { + return + } + l.hash = hash + + }) + + return err +} + +func (l *genericResource) initContent() error { var err error l.contentInit.Do(func() { - var b []byte + var r ReadSeekCloser + r, err = l.ReadSeekCloser() + if err != nil { + return + } + defer r.Close() - b, err := afero.ReadFile(l.sourceFs(), l.AbsSourceFilename()) + var b []byte + b, err = ioutil.ReadAll(r) if err != nil { return } @@ -462,7 +669,7 @@ func (l *genericResource) Content() (interface{}, error) { }) - return l.content, err + return err } func (l *genericResource) sourceFs() afero.Fs { @@ -472,12 +679,36 @@ func (l *genericResource) sourceFs() afero.Fs { return l.spec.sourceFs() } +func (l *genericResource) publishIfNeeded() { + if l.publishOnce != nil { + l.publishOnce.publish(l) + } +} + func (l *genericResource) Permalink() string { - return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetPath.path(), false), l.spec.BaseURL.String()) + l.publishIfNeeded() + return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetDirFile.path()), l.spec.BaseURL.HostURL()) } func (l *genericResource) RelPermalink() string { - return l.relPermalinkForRel(l.relTargetPath.path(), true) + l.publishIfNeeded() + return l.relPermalinkFor(l.relTargetDirFile.path()) +} + +func (l *genericResource) relPermalinkFor(target string) string { + return l.relPermalinkForRel(target) + +} +func (l *genericResource) permalinkFor(target string) string { + return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(target), l.spec.BaseURL.HostURL()) + +} +func (l *genericResource) relTargetPathFor(target string) string { + return l.relTargetPathForRel(target, false) +} + +func (l *genericResource) relTargetPath() string { + return l.relTargetPathForRel(l.targetPath(), false) } func (l *genericResource) Name() string { @@ -514,31 +745,33 @@ func (l *genericResource) updateParams(params map[string]interface{}) { } } -// Implement the Cloner interface. -func (l genericResource) WithNewBase(base string) Resource { - l.base = base - l.resourceContent = &resourceContent{} - return &l +func (l *genericResource) relPermalinkForRel(rel string) string { + return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, true)) } -func (l *genericResource) relPermalinkForRel(rel string, addBasePath bool) string { - return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, addBasePath)) -} +func (l *genericResource) relTargetPathForRel(rel string, isURL bool) string { -func (l *genericResource) relTargetPathForRel(rel string, addBasePath bool) string { if l.targetPathBuilder != nil { rel = l.targetPathBuilder(rel) } - if l.base != "" { - rel = path.Join(l.base, rel) + if isURL && l.baseURLDir != "" { + rel = path.Join(l.baseURLDir, rel) } - if addBasePath && l.spec.PathSpec.BasePath != "" { + if !isURL && l.baseTargetPathDir != "" { + rel = path.Join(l.baseTargetPathDir, rel) + } + + if l.baseOffset != "" { + rel = path.Join(l.baseOffset, rel) + } + + if isURL && l.spec.PathSpec.BasePath != "" { rel = path.Join(l.spec.PathSpec.BasePath, rel) } - if rel[0] != '/' { + if len(rel) == 0 || rel[0] != '/' { rel = "/" + rel } @@ -549,146 +782,100 @@ func (l *genericResource) ResourceType() string { return l.resourceType } -func (l *genericResource) AbsSourceFilename() string { - return l.sourceFilename -} - func (l *genericResource) String() string { return fmt.Sprintf("Resource(%s: %s)", l.resourceType, l.name) } func (l *genericResource) Publish() error { - f, err := l.sourceFs().Open(l.AbsSourceFilename()) + f, err := l.ReadSeekCloser() if err != nil { return err } defer f.Close() - return helpers.WriteToDisk(l.target(), f, l.spec.BaseFs.PublishFs) -} - -const counterPlaceHolder = ":counter" - -// AssignMetadata assigns the given metadata to those resources that supports updates -// and matching by wildcard given in `src` using `filepath.Match` with lower cased values. -// This assignment is additive, but the most specific match needs to be first. -// The `name` and `title` metadata field support shell-matched collection it got a match in. -// See https://golang.org/pkg/path/#Match -func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) error { - - counters := make(map[string]int) - - for _, r := range resources { - if _, ok := r.(metaAssigner); !ok { - continue - } - - var ( - nameSet, titleSet bool - nameCounter, titleCounter = 0, 0 - nameCounterFound, titleCounterFound bool - resourceSrcKey = strings.ToLower(r.Name()) - ) - - ma := r.(metaAssigner) - for _, meta := range metadata { - src, found := meta["src"] - if !found { - return fmt.Errorf("missing 'src' in metadata for resource") - } - - srcKey := strings.ToLower(cast.ToString(src)) - - glob, err := getGlob(srcKey) - if err != nil { - return fmt.Errorf("failed to match resource with metadata: %s", err) - } - - match := glob.Match(resourceSrcKey) - - if match { - if !nameSet { - name, found := meta["name"] - if found { - name := cast.ToString(name) - if !nameCounterFound { - nameCounterFound = strings.Contains(name, counterPlaceHolder) - } - if nameCounterFound && nameCounter == 0 { - counterKey := "name_" + srcKey - nameCounter = counters[counterKey] + 1 - counters[counterKey] = nameCounter - } - - ma.setName(replaceResourcePlaceholders(name, nameCounter)) - nameSet = true - } - } - - if !titleSet { - title, found := meta["title"] - if found { - title := cast.ToString(title) - if !titleCounterFound { - titleCounterFound = strings.Contains(title, counterPlaceHolder) - } - if titleCounterFound && titleCounter == 0 { - counterKey := "title_" + srcKey - titleCounter = counters[counterKey] + 1 - counters[counterKey] = titleCounter - } - ma.setTitle((replaceResourcePlaceholders(title, titleCounter))) - titleSet = true - } - } - - params, found := meta["params"] - if found { - m := cast.ToStringMap(params) - // Needed for case insensitive fetching of params values - maps.ToLower(m) - ma.updateParams(m) - } - } - } - } - - return nil + return helpers.WriteToDisk(l.targetFilename(), f, l.spec.BaseFs.PublishFs) } -func replaceResourcePlaceholders(in string, counter int) string { - return strings.Replace(in, counterPlaceHolder, strconv.Itoa(counter), -1) +// Path is stored with Unix style slashes. +func (l *genericResource) targetPath() string { + return l.relTargetDirFile.path() } -func (l *genericResource) target() string { - target := l.relTargetPathForRel(l.relTargetPath.path(), false) - if l.spec.PathSpec.Languages.IsMultihost() { - target = path.Join(l.spec.PathSpec.Language.Lang, target) - } - return filepath.Clean(target) +func (l *genericResource) targetFilename() string { + return filepath.Clean(l.relTargetPath()) } -func (r *Spec) newGenericResource( +// TODO(bep) clean up below +func (r *Spec) newGenericResource(sourceFs afero.Fs, + targetPathBuilder func(base string) string, + osFileInfo os.FileInfo, + sourceFilename, + baseFilename string, + mediaType media.Type) *genericResource { + return r.newGenericResourceWithBase( + sourceFs, + false, + nil, + "", + "", + targetPathBuilder, + osFileInfo, + sourceFilename, + baseFilename, + mediaType, + ) + +} + +func (r *Spec) newGenericResourceWithBase( + sourceFs afero.Fs, + lazyPublish bool, + openReadSeekerCloser OpenReadSeekCloser, + urlBaseDir string, + targetPathBaseDir string, targetPathBuilder func(base string) string, osFileInfo os.FileInfo, sourceFilename, - baseFilename, - resourceType string) *genericResource { + baseFilename string, + mediaType media.Type) *genericResource { // This value is used both to construct URLs and file paths, but start // with a Unix-styled path. - baseFilename = filepath.ToSlash(baseFilename) + baseFilename = helpers.ToSlashTrimLeading(baseFilename) fpath, fname := path.Split(baseFilename) - return &genericResource{ + var resourceType string + if mediaType.MainType == "image" { + resourceType = mediaType.MainType + } else { + resourceType = mediaType.SubType + } + + pathDescriptor := resourcePathDescriptor{ + baseURLDir: urlBaseDir, + baseTargetPathDir: targetPathBaseDir, targetPathBuilder: targetPathBuilder, - osFileInfo: osFileInfo, - sourceFilename: sourceFilename, - relTargetPath: dirFile{dir: fpath, file: fname}, - resourceType: resourceType, - spec: r, - params: make(map[string]interface{}), - name: baseFilename, - title: baseFilename, - resourceContent: &resourceContent{}, + relTargetDirFile: dirFile{dir: fpath, file: fname}, + } + + var po *publishOnce + if lazyPublish { + po = &publishOnce{logger: r.Logger} + } + + return &genericResource{ + openReadSeekerCloser: openReadSeekerCloser, + publishOnce: po, + resourcePathDescriptor: pathDescriptor, + overriddenSourceFs: sourceFs, + osFileInfo: osFileInfo, + sourceFilename: sourceFilename, + mediaType: mediaType, + resourceType: resourceType, + spec: r, + params: make(map[string]interface{}), + name: baseFilename, + title: baseFilename, + resourceContent: &resourceContent{}, + resourceHash: &resourceHash{}, } } diff --git a/resource/resource_cache.go b/resource/resource_cache.go new file mode 100644 index 000000000..28c3c23a2 --- /dev/null +++ b/resource/resource_cache.go @@ -0,0 +1,241 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "encoding/json" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "sync" + + "github.com/spf13/afero" + + "github.com/BurntSushi/locker" +) + +const ( + CACHE_CLEAR_ALL = "clear_all" + CACHE_OTHER = "other" +) + +type ResourceCache struct { + rs *Spec + + cache map[string]Resource + sync.RWMutex + + // Provides named resource locks. + nlocker *locker.Locker +} + +// ResourceKeyPartition returns a partition name +// to allow for more fine grained cache flushes. +// It will return the file extension without the leading ".". If no +// extension, it will return "other". +func ResourceKeyPartition(filename string) string { + ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(filename)), ".") + if ext == "" { + ext = CACHE_OTHER + } + return ext +} + +func newResourceCache(rs *Spec) *ResourceCache { + return &ResourceCache{ + rs: rs, + cache: make(map[string]Resource), + nlocker: locker.NewLocker(), + } +} + +func (c *ResourceCache) clear() { + c.Lock() + defer c.Unlock() + + c.cache = make(map[string]Resource) + c.nlocker = locker.NewLocker() +} + +func (c *ResourceCache) Contains(key string) bool { + key = c.cleanKey(filepath.ToSlash(key)) + _, found := c.get(key) + return found +} + +func (c *ResourceCache) cleanKey(key string) string { + return strings.TrimPrefix(path.Clean(key), "/") +} + +func (c *ResourceCache) get(key string) (Resource, bool) { + c.RLock() + defer c.RUnlock() + r, found := c.cache[key] + return r, found +} + +func (c *ResourceCache) GetOrCreate(partition, key string, f func() (Resource, error)) (Resource, error) { + key = c.cleanKey(path.Join(partition, key)) + // First check in-memory cache. + r, found := c.get(key) + if found { + return r, nil + } + // This is a potentially long running operation, so get a named lock. + c.nlocker.Lock(key) + + // Double check in-memory cache. + r, found = c.get(key) + if found { + c.nlocker.Unlock(key) + return r, nil + } + + defer c.nlocker.Unlock(key) + + r, err := f() + if err != nil { + return nil, err + } + + c.set(key, r) + + return r, nil + +} + +func (c *ResourceCache) getFilenames(key string) (string, string) { + filenameBase := filepath.Join(c.rs.GenAssetsPath, key) + filenameMeta := filenameBase + ".json" + filenameContent := filenameBase + ".content" + + return filenameMeta, filenameContent +} + +func (c *ResourceCache) getFromFile(key string) (afero.File, transformedResourceMetadata, bool) { + c.RLock() + defer c.RUnlock() + + var meta transformedResourceMetadata + filenameMeta, filenameContent := c.getFilenames(key) + fMeta, err := c.rs.Resources.Fs.Open(filenameMeta) + if err != nil { + return nil, meta, false + } + defer fMeta.Close() + + jsonContent, err := ioutil.ReadAll(fMeta) + if err != nil { + return nil, meta, false + } + + if err := json.Unmarshal(jsonContent, &meta); err != nil { + return nil, meta, false + } + + fContent, err := c.rs.Resources.Fs.Open(filenameContent) + if err != nil { + return nil, meta, false + } + + return fContent, meta, true +} + +// writeMeta writes the metadata to file and returns a writer for the content part. +func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata) (afero.File, error) { + filenameMeta, filenameContent := c.getFilenames(key) + raw, err := json.Marshal(meta) + if err != nil { + return nil, err + } + + fm, err := c.openResourceFileForWriting(filenameMeta) + if err != nil { + return nil, err + } + + if _, err := fm.Write(raw); err != nil { + return nil, err + } + + return c.openResourceFileForWriting(filenameContent) + +} + +func (c *ResourceCache) openResourceFileForWriting(filename string) (afero.File, error) { + return openFileForWriting(c.rs.Resources.Fs, filename) +} + +// openFileForWriting opens or creates the given file. If the target directory +// does not exist, it gets created. +func openFileForWriting(fs afero.Fs, filename string) (afero.File, error) { + filename = filepath.Clean(filename) + // Create will truncate if file already exists. + f, err := fs.Create(filename) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + if err = fs.MkdirAll(filepath.Dir(filename), 0755); err != nil { + return nil, err + } + f, err = fs.Create(filename) + } + + return f, err +} + +func (c *ResourceCache) set(key string, r Resource) { + c.Lock() + defer c.Unlock() + c.cache[key] = r +} + +func (c *ResourceCache) DeletePartitions(partitions ...string) { + partitionsSet := map[string]bool{ + // Always clear out the resources not matching the partition. + "other": true, + } + for _, p := range partitions { + partitionsSet[p] = true + } + + if partitionsSet[CACHE_CLEAR_ALL] { + c.clear() + return + } + + c.Lock() + defer c.Unlock() + + for k := range c.cache { + clear := false + partIdx := strings.Index(k, "/") + if partIdx == -1 { + clear = true + } else { + partition := k[:partIdx] + if partitionsSet[partition] { + clear = true + } + } + + if clear { + delete(c.cache, k) + } + } + +} diff --git a/resource/resource_metadata.go b/resource/resource_metadata.go new file mode 100644 index 000000000..2c82aeaf6 --- /dev/null +++ b/resource/resource_metadata.go @@ -0,0 +1,129 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "fmt" + "strconv" + + "github.com/spf13/cast" + + "strings" + + "github.com/gohugoio/hugo/common/maps" +) + +var ( + _ metaAssigner = (*genericResource)(nil) +) + +// metaAssigner allows updating metadata in resources that supports it. +type metaAssigner interface { + setTitle(title string) + setName(name string) + updateParams(params map[string]interface{}) +} + +const counterPlaceHolder = ":counter" + +// AssignMetadata assigns the given metadata to those resources that supports updates +// and matching by wildcard given in `src` using `filepath.Match` with lower cased values. +// This assignment is additive, but the most specific match needs to be first. +// The `name` and `title` metadata field support shell-matched collection it got a match in. +// See https://golang.org/pkg/path/#Match +func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) error { + + counters := make(map[string]int) + + for _, r := range resources { + if _, ok := r.(metaAssigner); !ok { + continue + } + + var ( + nameSet, titleSet bool + nameCounter, titleCounter = 0, 0 + nameCounterFound, titleCounterFound bool + resourceSrcKey = strings.ToLower(r.Name()) + ) + + ma := r.(metaAssigner) + for _, meta := range metadata { + src, found := meta["src"] + if !found { + return fmt.Errorf("missing 'src' in metadata for resource") + } + + srcKey := strings.ToLower(cast.ToString(src)) + + glob, err := getGlob(srcKey) + if err != nil { + return fmt.Errorf("failed to match resource with metadata: %s", err) + } + + match := glob.Match(resourceSrcKey) + + if match { + if !nameSet { + name, found := meta["name"] + if found { + name := cast.ToString(name) + if !nameCounterFound { + nameCounterFound = strings.Contains(name, counterPlaceHolder) + } + if nameCounterFound && nameCounter == 0 { + counterKey := "name_" + srcKey + nameCounter = counters[counterKey] + 1 + counters[counterKey] = nameCounter + } + + ma.setName(replaceResourcePlaceholders(name, nameCounter)) + nameSet = true + } + } + + if !titleSet { + title, found := meta["title"] + if found { + title := cast.ToString(title) + if !titleCounterFound { + titleCounterFound = strings.Contains(title, counterPlaceHolder) + } + if titleCounterFound && titleCounter == 0 { + counterKey := "title_" + srcKey + titleCounter = counters[counterKey] + 1 + counters[counterKey] = titleCounter + } + ma.setTitle((replaceResourcePlaceholders(title, titleCounter))) + titleSet = true + } + } + + params, found := meta["params"] + if found { + m := cast.ToStringMap(params) + // Needed for case insensitive fetching of params values + maps.ToLower(m) + ma.updateParams(m) + } + } + } + } + + return nil +} + +func replaceResourcePlaceholders(in string, counter int) string { + return strings.Replace(in, counterPlaceHolder, strconv.Itoa(counter), -1) +} diff --git a/resource/resource_metadata_test.go b/resource/resource_metadata_test.go new file mode 100644 index 000000000..85fb25b57 --- /dev/null +++ b/resource/resource_metadata_test.go @@ -0,0 +1,230 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "testing" + + "github.com/gohugoio/hugo/media" + + "github.com/stretchr/testify/require" +) + +func TestAssignMetadata(t *testing.T) { + assert := require.New(t) + spec := newTestResourceSpec(assert) + + var foo1, foo2, foo3, logo1, logo2, logo3 Resource + var resources Resources + + for _, this := range []struct { + metaData []map[string]interface{} + assertFunc func(err error) + }{ + {[]map[string]interface{}{ + { + "title": "My Resource", + "name": "My Name", + "src": "*", + }, + }, func(err error) { + assert.Equal("My Resource", logo1.Title()) + assert.Equal("My Name", logo1.Name()) + assert.Equal("My Name", foo2.Name()) + + }}, + {[]map[string]interface{}{ + { + "title": "My Logo", + "src": "*loGo*", + }, + { + "title": "My Resource", + "name": "My Name", + "src": "*", + }, + }, func(err error) { + assert.Equal("My Logo", logo1.Title()) + assert.Equal("My Logo", logo2.Title()) + assert.Equal("My Name", logo1.Name()) + assert.Equal("My Name", foo2.Name()) + assert.Equal("My Name", foo3.Name()) + assert.Equal("My Resource", foo3.Title()) + + }}, + {[]map[string]interface{}{ + { + "title": "My Logo", + "src": "*loGo*", + "params": map[string]interface{}{ + "Param1": true, + "icon": "logo", + }, + }, + { + "title": "My Resource", + "src": "*", + "params": map[string]interface{}{ + "Param2": true, + "icon": "resource", + }, + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("My Logo", logo1.Title()) + assert.Equal("My Resource", foo3.Title()) + _, p1 := logo2.Params()["param1"] + _, p2 := foo2.Params()["param2"] + _, p1_2 := foo2.Params()["param1"] + _, p2_2 := logo2.Params()["param2"] + + icon1, _ := logo2.Params()["icon"] + icon2, _ := foo2.Params()["icon"] + + assert.True(p1) + assert.True(p2) + + // Check merge + assert.True(p2_2) + assert.False(p1_2) + + assert.Equal("logo", icon1) + assert.Equal("resource", icon2) + + }}, + {[]map[string]interface{}{ + { + "name": "Logo Name #:counter", + "src": "*logo*", + }, + { + "title": "Resource #:counter", + "name": "Name #:counter", + "src": "*", + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("Resource #2", logo2.Title()) + assert.Equal("Logo Name #1", logo2.Name()) + assert.Equal("Resource #4", logo1.Title()) + assert.Equal("Logo Name #2", logo1.Name()) + assert.Equal("Resource #1", foo2.Title()) + assert.Equal("Resource #3", foo1.Title()) + assert.Equal("Name #2", foo1.Name()) + assert.Equal("Resource #5", foo3.Title()) + + assert.Equal(logo2, resources.GetMatch("logo name #1*")) + + }}, + {[]map[string]interface{}{ + { + "title": "Third Logo #:counter", + "src": "logo3.png", + }, + { + "title": "Other Logo #:counter", + "name": "Name #:counter", + "src": "logo*", + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("Third Logo #1", logo3.Title()) + assert.Equal("Name #3", logo3.Name()) + assert.Equal("Other Logo #1", logo2.Title()) + assert.Equal("Name #1", logo2.Name()) + assert.Equal("Other Logo #2", logo1.Title()) + assert.Equal("Name #2", logo1.Name()) + + }}, + {[]map[string]interface{}{ + { + "title": "Third Logo", + "src": "logo3.png", + }, + { + "title": "Other Logo #:counter", + "name": "Name #:counter", + "src": "logo*", + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("Third Logo", logo3.Title()) + assert.Equal("Name #3", logo3.Name()) + assert.Equal("Other Logo #1", logo2.Title()) + assert.Equal("Name #1", logo2.Name()) + assert.Equal("Other Logo #2", logo1.Title()) + assert.Equal("Name #2", logo1.Name()) + + }}, + {[]map[string]interface{}{ + { + "name": "third-logo", + "src": "logo3.png", + }, + { + "title": "Logo #:counter", + "name": "Name #:counter", + "src": "logo*", + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("Logo #3", logo3.Title()) + assert.Equal("third-logo", logo3.Name()) + assert.Equal("Logo #1", logo2.Title()) + assert.Equal("Name #1", logo2.Name()) + assert.Equal("Logo #2", logo1.Title()) + assert.Equal("Name #2", logo1.Name()) + + }}, + {[]map[string]interface{}{ + { + "title": "Third Logo #:counter", + }, + }, func(err error) { + // Missing src + assert.Error(err) + + }}, + {[]map[string]interface{}{ + { + "title": "Title", + "src": "[]", + }, + }, func(err error) { + // Invalid pattern + assert.Error(err) + + }}, + } { + + foo2 = spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType) + logo2 = spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType) + foo1 = spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType) + logo1 = spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType) + foo3 = spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType) + logo3 = spec.newGenericResource(nil, nil, nil, "/b/logo3.png", "logo3.png", pngType) + + resources = Resources{ + foo2, + logo2, + foo1, + logo1, + foo3, + logo3, + } + + this.assertFunc(AssignMetadata(this.metaData, resources...)) + } + +} diff --git a/resource/resource_test.go b/resource/resource_test.go index 40061e5c4..659994c36 100644 --- a/resource/resource_test.go +++ b/resource/resource_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,6 +22,8 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/media" + "github.com/stretchr/testify/require" ) @@ -29,7 +31,7 @@ func TestGenericResource(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) - r := spec.newGenericResource(nil, nil, "/a/foo.css", "foo.css", "css") + r := spec.newGenericResource(nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType) assert.Equal("https://example.com/foo.css", r.Permalink()) assert.Equal("/foo.css", r.RelPermalink()) @@ -44,7 +46,7 @@ func TestGenericResourceWithLinkFacory(t *testing.T) { factory := func(s string) string { return path.Join("/foo", s) } - r := spec.newGenericResource(factory, nil, "/a/foo.css", "foo.css", "css") + r := spec.newGenericResource(nil, factory, nil, "/a/foo.css", "foo.css", media.CSSType) assert.Equal("https://example.com/foo/foo.css", r.Permalink()) assert.Equal("/foo/foo.css", r.RelPermalink()) @@ -58,8 +60,7 @@ func TestNewResourceFromFilename(t *testing.T) { writeSource(t, spec.Fs, "content/a/b/logo.png", "image") writeSource(t, spec.Fs, "content/a/b/data.json", "json") - r, err := spec.NewResourceFromFilename(nil, - filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png")) + r, err := spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/logo.png"}) assert.NoError(err) assert.NotNil(r) @@ -67,7 +68,7 @@ func TestNewResourceFromFilename(t *testing.T) { assert.Equal("/a/b/logo.png", r.RelPermalink()) assert.Equal("https://example.com/a/b/logo.png", r.Permalink()) - r, err = spec.NewResourceFromFilename(nil, "a/b/data.json", "a/b/data.json") + r, err = spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/data.json"}) assert.NoError(err) assert.NotNil(r) @@ -84,8 +85,7 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) { writeSource(t, spec.Fs, "content/a/b/logo.png", "image") - r, err := spec.NewResourceFromFilename(nil, - filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png")) + r, err := spec.New(ResourceSourceDescriptor{SourceFilename: filepath.FromSlash("a/b/logo.png")}) assert.NoError(err) assert.NotNil(r) @@ -93,18 +93,20 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) { assert.Equal("/docs/a/b/logo.png", r.RelPermalink()) assert.Equal("https://example.com/docs/a/b/logo.png", r.Permalink()) img := r.(*Image) - assert.Equal(filepath.FromSlash("/a/b/logo.png"), img.target()) + assert.Equal(filepath.FromSlash("/a/b/logo.png"), img.targetFilename()) } +var pngType, _ = media.FromString("image/png") + func TestResourcesByType(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) resources := Resources{ - spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"), - spec.newGenericResource(nil, nil, "/a/logo.png", "logo.css", "image"), - spec.newGenericResource(nil, nil, "/a/foo2.css", "foo2.css", "css"), - spec.newGenericResource(nil, nil, "/a/foo3.css", "foo3.css", "css")} + spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/a/logo.png", "logo.css", pngType), + spec.newGenericResource(nil, nil, nil, "/a/foo2.css", "foo2.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/a/foo3.css", "foo3.css", media.CSSType)} assert.Len(resources.ByType("css"), 3) assert.Len(resources.ByType("image"), 1) @@ -115,25 +117,25 @@ func TestResourcesGetByPrefix(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) resources := Resources{ - spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"), - spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"), - spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"), - spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"), - spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css")} - - assert.Nil(resources.GetByPrefix("asdf")) - assert.Equal("/logo1.png", resources.GetByPrefix("logo").RelPermalink()) - assert.Equal("/logo1.png", resources.GetByPrefix("loGo").RelPermalink()) - assert.Equal("/Logo2.png", resources.GetByPrefix("logo2").RelPermalink()) - assert.Equal("/foo2.css", resources.GetByPrefix("foo2").RelPermalink()) - assert.Equal("/foo1.css", resources.GetByPrefix("foo1").RelPermalink()) - assert.Equal("/foo1.css", resources.GetByPrefix("foo1").RelPermalink()) - assert.Nil(resources.GetByPrefix("asdfasdf")) - - assert.Equal(2, len(resources.ByPrefix("logo"))) - assert.Equal(1, len(resources.ByPrefix("logo2"))) - - logo := resources.GetByPrefix("logo") + spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)} + + assert.Nil(resources.GetMatch("asdf*")) + assert.Equal("/logo1.png", resources.GetMatch("logo*").RelPermalink()) + assert.Equal("/logo1.png", resources.GetMatch("loGo*").RelPermalink()) + assert.Equal("/Logo2.png", resources.GetMatch("logo2*").RelPermalink()) + assert.Equal("/foo2.css", resources.GetMatch("foo2*").RelPermalink()) + assert.Equal("/foo1.css", resources.GetMatch("foo1*").RelPermalink()) + assert.Equal("/foo1.css", resources.GetMatch("foo1*").RelPermalink()) + assert.Nil(resources.GetMatch("asdfasdf*")) + + assert.Equal(2, len(resources.Match("logo*"))) + assert.Equal(1, len(resources.Match("logo2*"))) + + logo := resources.GetMatch("logo*") assert.NotNil(logo.Params()) assert.Equal("logo1.png", logo.Name()) assert.Equal("logo1.png", logo.Title()) @@ -144,14 +146,14 @@ func TestResourcesGetMatch(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) resources := Resources{ - spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"), - spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"), - spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"), - spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"), - spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css"), - spec.newGenericResource(nil, nil, "/b/c/foo4.css", "c/foo4.css", "css"), - spec.newGenericResource(nil, nil, "/b/c/foo5.css", "c/foo5.css", "css"), - spec.newGenericResource(nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", "css"), + spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/c/foo4.css", "c/foo4.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/c/foo5.css", "c/foo5.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", media.CSSType), } assert.Equal("/logo1.png", resources.GetMatch("logo*").RelPermalink()) @@ -186,226 +188,6 @@ func TestResourcesGetMatch(t *testing.T) { } -func TestAssignMetadata(t *testing.T) { - assert := require.New(t) - spec := newTestResourceSpec(assert) - - var foo1, foo2, foo3, logo1, logo2, logo3 Resource - var resources Resources - - for _, this := range []struct { - metaData []map[string]interface{} - assertFunc func(err error) - }{ - {[]map[string]interface{}{ - { - "title": "My Resource", - "name": "My Name", - "src": "*", - }, - }, func(err error) { - assert.Equal("My Resource", logo1.Title()) - assert.Equal("My Name", logo1.Name()) - assert.Equal("My Name", foo2.Name()) - - }}, - {[]map[string]interface{}{ - { - "title": "My Logo", - "src": "*loGo*", - }, - { - "title": "My Resource", - "name": "My Name", - "src": "*", - }, - }, func(err error) { - assert.Equal("My Logo", logo1.Title()) - assert.Equal("My Logo", logo2.Title()) - assert.Equal("My Name", logo1.Name()) - assert.Equal("My Name", foo2.Name()) - assert.Equal("My Name", foo3.Name()) - assert.Equal("My Resource", foo3.Title()) - - }}, - {[]map[string]interface{}{ - { - "title": "My Logo", - "src": "*loGo*", - "params": map[string]interface{}{ - "Param1": true, - "icon": "logo", - }, - }, - { - "title": "My Resource", - "src": "*", - "params": map[string]interface{}{ - "Param2": true, - "icon": "resource", - }, - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("My Logo", logo1.Title()) - assert.Equal("My Resource", foo3.Title()) - _, p1 := logo2.Params()["param1"] - _, p2 := foo2.Params()["param2"] - _, p1_2 := foo2.Params()["param1"] - _, p2_2 := logo2.Params()["param2"] - - icon1, _ := logo2.Params()["icon"] - icon2, _ := foo2.Params()["icon"] - - assert.True(p1) - assert.True(p2) - - // Check merge - assert.True(p2_2) - assert.False(p1_2) - - assert.Equal("logo", icon1) - assert.Equal("resource", icon2) - - }}, - {[]map[string]interface{}{ - { - "name": "Logo Name #:counter", - "src": "*logo*", - }, - { - "title": "Resource #:counter", - "name": "Name #:counter", - "src": "*", - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("Resource #2", logo2.Title()) - assert.Equal("Logo Name #1", logo2.Name()) - assert.Equal("Resource #4", logo1.Title()) - assert.Equal("Logo Name #2", logo1.Name()) - assert.Equal("Resource #1", foo2.Title()) - assert.Equal("Resource #3", foo1.Title()) - assert.Equal("Name #2", foo1.Name()) - assert.Equal("Resource #5", foo3.Title()) - - assert.Equal(logo2, resources.GetByPrefix("logo name #1")) - - }}, - {[]map[string]interface{}{ - { - "title": "Third Logo #:counter", - "src": "logo3.png", - }, - { - "title": "Other Logo #:counter", - "name": "Name #:counter", - "src": "logo*", - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("Third Logo #1", logo3.Title()) - assert.Equal("Name #3", logo3.Name()) - assert.Equal("Other Logo #1", logo2.Title()) - assert.Equal("Name #1", logo2.Name()) - assert.Equal("Other Logo #2", logo1.Title()) - assert.Equal("Name #2", logo1.Name()) - - }}, - {[]map[string]interface{}{ - { - "title": "Third Logo", - "src": "logo3.png", - }, - { - "title": "Other Logo #:counter", - "name": "Name #:counter", - "src": "logo*", - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("Third Logo", logo3.Title()) - assert.Equal("Name #3", logo3.Name()) - assert.Equal("Other Logo #1", logo2.Title()) - assert.Equal("Name #1", logo2.Name()) - assert.Equal("Other Logo #2", logo1.Title()) - assert.Equal("Name #2", logo1.Name()) - - }}, - {[]map[string]interface{}{ - { - "name": "third-logo", - "src": "logo3.png", - }, - { - "title": "Logo #:counter", - "name": "Name #:counter", - "src": "logo*", - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("Logo #3", logo3.Title()) - assert.Equal("third-logo", logo3.Name()) - assert.Equal("Logo #1", logo2.Title()) - assert.Equal("Name #1", logo2.Name()) - assert.Equal("Logo #2", logo1.Title()) - assert.Equal("Name #2", logo1.Name()) - - }}, - {[]map[string]interface{}{ - { - "title": "Third Logo #:counter", - }, - }, func(err error) { - // Missing src - assert.Error(err) - - }}, - {[]map[string]interface{}{ - { - "title": "Title", - "src": "[]", - }, - }, func(err error) { - // Invalid pattern - assert.Error(err) - - }}, - } { - - foo2 = spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css") - logo2 = spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image") - foo1 = spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css") - logo1 = spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image") - foo3 = spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css") - logo3 = spec.newGenericResource(nil, nil, "/b/logo3.png", "logo3.png", "image") - - resources = Resources{ - foo2, - logo2, - foo1, - logo1, - foo3, - logo3, - } - - this.assertFunc(AssignMetadata(this.metaData, resources...)) - } - -} - -func BenchmarkResourcesByPrefix(b *testing.B) { - resources := benchResources(b) - prefixes := []string{"abc", "jkl", "nomatch", "sub/"} - rnd := rand.New(rand.NewSource(time.Now().Unix())) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - resources.ByPrefix(prefixes[rnd.Intn(len(prefixes))]) - } - }) -} - func BenchmarkResourcesMatch(b *testing.B) { resources := benchResources(b) prefixes := []string{"abc*", "jkl*", "nomatch*", "sub/*"} @@ -428,7 +210,7 @@ func BenchmarkResourcesMatchA100(b *testing.B) { a100 := strings.Repeat("a", 100) pattern := "a*a*a*a*a*a*a*a*b" - resources := Resources{spec.newGenericResource(nil, nil, "/a/"+a100, a100, "css")} + resources := Resources{spec.newGenericResource(nil, nil, nil, "/a/"+a100, a100, media.CSSType)} b.ResetTimer() for i := 0; i < b.N; i++ { @@ -444,17 +226,17 @@ func benchResources(b *testing.B) Resources { for i := 0; i < 30; i++ { name := fmt.Sprintf("abcde%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("efghi%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("jklmn%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/b/sub/"+name, "sub/"+name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, nil, "/b/sub/"+name, "sub/"+name, media.CSSType)) } return resources @@ -482,7 +264,7 @@ func BenchmarkAssignMetadata(b *testing.B) { } for i := 0; i < 20; i++ { name := fmt.Sprintf("foo%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) } b.StartTimer() diff --git a/resource/templates/execute_as_template.go b/resource/templates/execute_as_template.go new file mode 100644 index 000000000..dee9d0d9a --- /dev/null +++ b/resource/templates/execute_as_template.go @@ -0,0 +1,76 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package templates contains functions for template processing of Resource objects. +package templates + +import ( + "fmt" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/resource" + "github.com/gohugoio/hugo/tpl" +) + +// Client contains methods to perform template processing of Resource objects. +type Client struct { + rs *resource.Spec + + textTemplate tpl.TemplateParseFinder +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec, textTemplate tpl.TemplateParseFinder) *Client { + if rs == nil { + panic("must provice a resource Spec") + } + if textTemplate == nil { + panic("must provide a textTemplate") + } + return &Client{rs: rs, textTemplate: textTemplate} +} + +type executeAsTemplateTransform struct { + rs *resource.Spec + textTemplate tpl.TemplateParseFinder + targetPath string + data interface{} +} + +func (t *executeAsTemplateTransform) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("execute-as-template", t.targetPath) +} + +func (t *executeAsTemplateTransform) Transform(ctx *resource.ResourceTransformationCtx) error { + tplStr := helpers.ReaderToString(ctx.From) + templ, err := t.textTemplate.Parse(ctx.InPath, tplStr) + if err != nil { + return fmt.Errorf("failed to parse Resource %q as Template: %s", ctx.InPath, err) + } + + ctx.OutPath = t.targetPath + + return templ.Execute(ctx.To, t.data) +} + +func (c *Client) ExecuteAsTemplate(res resource.Resource, targetPath string, data interface{}) (resource.Resource, error) { + return c.rs.Transform( + res, + &executeAsTemplateTransform{ + rs: c.rs, + targetPath: helpers.ToSlashTrimLeading(targetPath), + textTemplate: c.textTemplate, + data: data, + }, + ) +} diff --git a/resource/testhelpers_test.go b/resource/testhelpers_test.go index 360adc038..e78a536a2 100644 --- a/resource/testhelpers_test.go +++ b/resource/testhelpers_test.go @@ -33,7 +33,9 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) * cfg.Set("dataDir", "data") cfg.Set("i18nDir", "i18n") cfg.Set("layoutDir", "layouts") + cfg.Set("assetDir", "assets") cfg.Set("archetypeDir", "archetypes") + cfg.Set("publishDir", "public") imagingCfg := map[string]interface{}{ "resampleFilter": "linear", @@ -49,7 +51,7 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) * assert.NoError(err) - spec, err := NewSpec(s, media.DefaultTypes) + spec, err := NewSpec(s, nil, media.DefaultTypes) assert.NoError(err) return spec } @@ -72,7 +74,9 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { cfg.Set("dataDir", "data") cfg.Set("i18nDir", "i18n") cfg.Set("layoutDir", "layouts") + cfg.Set("assetDir", "assets") cfg.Set("archetypeDir", "archetypes") + cfg.Set("publishDir", "public") fs := hugofs.NewFrom(hugofs.Os, cfg) fs.Destination = &afero.MemMapFs{} @@ -81,7 +85,7 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { assert.NoError(err) - spec, err := NewSpec(s, media.DefaultTypes) + spec, err := NewSpec(s, nil, media.DefaultTypes) assert.NoError(err) return spec @@ -102,12 +106,11 @@ func fetchImageForSpec(spec *Spec, assert *require.Assertions, name string) *Ima return r.(*Image) } -func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) Resource { +func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) ContentResource { src, err := os.Open(filepath.FromSlash("testdata/" + name)) assert.NoError(err) - assert.NoError(spec.BaseFs.ContentFs.MkdirAll(filepath.Dir(name), 0755)) - out, err := spec.BaseFs.ContentFs.Create(name) + out, err := openFileForWriting(spec.BaseFs.Content.Fs, name) assert.NoError(err) _, err = io.Copy(out, src) out.Close() @@ -118,10 +121,10 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) R return path.Join("/a", s) } - r, err := spec.NewResourceFromFilename(factory, name, name) + r, err := spec.New(ResourceSourceDescriptor{TargetPathBuilder: factory, SourceFilename: name}) assert.NoError(err) - return r + return r.(ContentResource) } func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, width, height int) { diff --git a/resource/tocss/scss/client.go b/resource/tocss/scss/client.go new file mode 100644 index 000000000..610ea3845 --- /dev/null +++ b/resource/tocss/scss/client.go @@ -0,0 +1,101 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scss + +import ( + "github.com/bep/go-tocss/scss" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/resource" + "github.com/mitchellh/mapstructure" +) + +type Client struct { + rs *resource.Spec + sfs *filesystems.SourceFilesystem +} + +func New(fs *filesystems.SourceFilesystem, rs *resource.Spec) (*Client, error) { + return &Client{sfs: fs, rs: rs}, nil +} + +type Options struct { + + // Hugo, will by default, just replace the extension of the source + // to .css, e.g. "scss/main.scss" becomes "scss/main.css". You can + // control this by setting this, e.g. "styles/main.css" will create + // a Resource with that as a base for RelPermalink etc. + TargetPath string + + // Default is nested. + // One of nested, expanded, compact, compressed. + OutputStyle string + + // Precision of floating point math. + Precision int + + // When enabled, Hugo will generate a source map. + EnableSourceMap bool +} + +type options struct { + // The options we receive from the end user. + from Options + + // The options we send to the SCSS library. + to scss.Options +} + +func (c *Client) ToCSS(res resource.Resource, opts Options) (resource.Resource, error) { + internalOptions := options{ + from: opts, + } + + // Transfer values from client. + internalOptions.to.Precision = opts.Precision + internalOptions.to.OutputStyle = scss.OutputStyleFromString(opts.OutputStyle) + + if internalOptions.to.Precision == 0 { + // bootstrap-sass requires 8 digits precision. The libsass default is 5. + // https://github.com/twbs/bootstrap-sass/blob/master/README.md#sass-number-precision + internalOptions.to.Precision = 8 + } + + return c.rs.Transform( + res, + &toCSSTransformation{c: c, options: internalOptions}, + ) +} + +type toCSSTransformation struct { + c *Client + options options +} + +func (t *toCSSTransformation) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("tocss", t.options.from) +} + +func DecodeOptions(m map[string]interface{}) (opts Options, err error) { + if m == nil { + return + } + err = mapstructure.WeakDecode(m, &opts) + + if opts.TargetPath != "" { + opts.TargetPath = helpers.ToSlashTrimLeading(opts.TargetPath) + } + + return +} diff --git a/resource/tocss/scss/tocss.go b/resource/tocss/scss/tocss.go new file mode 100644 index 000000000..d606e9832 --- /dev/null +++ b/resource/tocss/scss/tocss.go @@ -0,0 +1,111 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build extended + +package scss + +import ( + "fmt" + "io" + "path" + "strings" + + "github.com/bep/go-tocss/scss" + "github.com/bep/go-tocss/scss/libsass" + "github.com/bep/go-tocss/tocss" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resource" +) + +// Used in tests. This feature requires Hugo to be built with the extended tag. +func Supports() bool { + return true +} + +func (t *toCSSTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + ctx.OutMediaType = media.CSSType + + var outName string + if t.options.from.TargetPath != "" { + ctx.OutPath = t.options.from.TargetPath + } else { + ctx.ReplaceOutPathExtension(".css") + } + + outName = path.Base(ctx.OutPath) + + options := t.options + + // We may allow the end user to add IncludePaths later, if we find a use + // case for that. + options.to.IncludePaths = t.c.sfs.RealDirs(path.Dir(ctx.SourcePath)) + + if ctx.InMediaType.SubType == media.SASSType.SubType { + options.to.SassSyntax = true + } + + if options.from.EnableSourceMap { + + options.to.SourceMapFilename = outName + ".map" + options.to.SourceMapRoot = t.c.rs.WorkingDir + + // Setting this to the relative input filename will get the source map + // more correct for the main entry path (main.scss typically), but + // it will mess up the import mappings. As a workaround, we do a replacement + // in the source map itself (see below). + //options.InputPath = inputPath + options.to.OutputPath = outName + options.to.SourceMapContents = true + options.to.OmitSourceMapURL = false + options.to.EnableEmbeddedSourceMap = false + } + + res, err := t.c.toCSS(options.to, ctx.To, ctx.From) + if err != nil { + return err + } + + if options.from.EnableSourceMap && res.SourceMapContent != "" { + sourcePath := t.c.sfs.RealFilename(ctx.SourcePath) + + if strings.HasPrefix(sourcePath, t.c.rs.WorkingDir) { + sourcePath = strings.TrimPrefix(sourcePath, t.c.rs.WorkingDir+helpers.FilePathSeparator) + } + + // This is a workaround for what looks like a bug in Libsass. But + // getting this resolution correct in tools like Chrome Workspaces + // is important enough to go this extra mile. + mapContent := strings.Replace(res.SourceMapContent, `stdin",`, fmt.Sprintf("%s\",", sourcePath), 1) + + return ctx.PublishSourceMap(mapContent) + } + return nil +} + +func (c *Client) toCSS(options scss.Options, dst io.Writer, src io.Reader) (tocss.Result, error) { + var res tocss.Result + + transpiler, err := libsass.New(options) + if err != nil { + return res, err + } + + res, err = transpiler.Execute(dst, src) + if err != nil { + return res, fmt.Errorf("SCSS processing failed: %s", err) + } + + return res, nil +} diff --git a/resource/tocss/scss/tocss_notavailable.go b/resource/tocss/scss/tocss_notavailable.go new file mode 100644 index 000000000..69b4fc655 --- /dev/null +++ b/resource/tocss/scss/tocss_notavailable.go @@ -0,0 +1,30 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !extended + +package scss + +import ( + "github.com/gohugoio/hugo/common/errors" + "github.com/gohugoio/hugo/resource" +) + +// Used in tests. +func Supports() bool { + return false +} + +func (t *toCSSTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + return errors.FeatureNotAvailableErr +} diff --git a/resource/transform.go b/resource/transform.go new file mode 100644 index 000000000..6a100ddc4 --- /dev/null +++ b/resource/transform.go @@ -0,0 +1,487 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "bytes" + "path" + "strconv" + + "github.com/gohugoio/hugo/common/errors" + "github.com/gohugoio/hugo/helpers" + "github.com/mitchellh/hashstructure" + "github.com/spf13/afero" + + "fmt" + "io" + "sync" + + "github.com/gohugoio/hugo/media" + + bp "github.com/gohugoio/hugo/bufferpool" +) + +var ( + _ ContentResource = (*transformedResource)(nil) + _ ReadSeekCloserResource = (*transformedResource)(nil) +) + +func (s *Spec) Transform(r Resource, t ResourceTransformation) (Resource, error) { + return &transformedResource{ + Resource: r, + transformation: t, + transformedResourceMetadata: transformedResourceMetadata{MetaData: make(map[string]interface{})}, + cache: s.ResourceCache}, nil +} + +type ResourceTransformationCtx struct { + // The content to transform. + From io.Reader + + // The target of content transformation. + // The current implementation requires that r is written to w + // even if no transformation is performed. + To io.Writer + + // This is the relative path to the original source. Unix styled slashes. + SourcePath string + + // This is the relative target path to the resource. Unix styled slashes. + InPath string + + // The relative target path to the transformed resource. Unix styled slashes. + OutPath string + + // The input media type + InMediaType media.Type + + // The media type of the transformed resource. + OutMediaType media.Type + + // Data data can be set on the transformed Resource. Not that this need + // to be simple types, as it needs to be serialized to JSON and back. + Data map[string]interface{} + + // This is used to publis additional artifacts, e.g. source maps. + // We may improve this. + OpenResourcePublisher func(relTargetPath string) (io.WriteCloser, error) +} + +// AddOutPathIdentifier transforming InPath to OutPath adding an identifier, +// eg '.min' before any extension. +func (ctx *ResourceTransformationCtx) AddOutPathIdentifier(identifier string) { + ctx.OutPath = ctx.addPathIdentifier(ctx.InPath, identifier) +} + +func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string { + dir, file := path.Split(inPath) + base, ext := helpers.PathAndExt(file) + return path.Join(dir, (base + identifier + ext)) +} + +// ReplaceOutPathExtension transforming InPath to OutPath replacing the file +// extension, e.g. ".scss" +func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) { + dir, file := path.Split(ctx.InPath) + base, _ := helpers.PathAndExt(file) + ctx.OutPath = path.Join(dir, (base + newExt)) +} + +// PublishSourceMap writes the content to the target folder of the main resource +// with the ".map" extension added. +func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error { + target := ctx.OutPath + ".map" + f, err := ctx.OpenResourcePublisher(target) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write([]byte(content)) + return err +} + +// ResourceTransformationKey are provided by the different transformation implementations. +// It identifies the transformation (name) and its configuration (elements). +// We combine this in a chain with the rest of the transformations +// with the target filename and a content hash of the origin to use as cache key. +type ResourceTransformationKey struct { + name string + elements []interface{} +} + +// NewResourceTransformationKey creates a new ResourceTransformationKey from the transformation +// name and elements. We will create a 64 bit FNV hash from the elements, which when combined +// with the other key elements should be unique for all practical applications. +func NewResourceTransformationKey(name string, elements ...interface{}) ResourceTransformationKey { + return ResourceTransformationKey{name: name, elements: elements} +} + +// Do not change this without good reasons. +func (k ResourceTransformationKey) key() string { + if len(k.elements) == 0 { + return k.name + } + + sb := bp.GetBuffer() + defer bp.PutBuffer(sb) + + sb.WriteString(k.name) + for _, element := range k.elements { + hash, err := hashstructure.Hash(element, nil) + if err != nil { + panic(err) + } + sb.WriteString("_") + sb.WriteString(strconv.FormatUint(hash, 10)) + } + + return sb.String() +} + +// ResourceTransformation is the interface that a resource transformation step +// needs to implement. +type ResourceTransformation interface { + Key() ResourceTransformationKey + Transform(ctx *ResourceTransformationCtx) error +} + +// We will persist this information to disk. +type transformedResourceMetadata struct { + Target string `json:"Target"` + MediaTypeV string `json:"MediaType"` + MetaData map[string]interface{} `json:"Data"` +} + +type transformedResource struct { + cache *ResourceCache + + // This is the filename inside resources/_gen/assets + sourceFilename string + + linker permalinker + + // The transformation to apply. + transformation ResourceTransformation + + // We apply the tranformations lazily. + transformInit sync.Once + transformErr error + + // The transformed values + content string + contentInit sync.Once + transformedResourceMetadata + + // The source + Resource +} + +func (r *transformedResource) ReadSeekCloser() (ReadSeekCloser, error) { + rc, ok := r.Resource.(ReadSeekCloserResource) + if !ok { + return nil, fmt.Errorf("resource %T is not a ReadSeekerCloserResource", rc) + } + return rc.ReadSeekCloser() +} + +func (r *transformedResource) transferTransformedValues(another *transformedResource) { + if another.content != "" { + r.contentInit.Do(func() { + r.content = another.content + }) + } + r.transformedResourceMetadata = another.transformedResourceMetadata +} + +func (r *transformedResource) tryTransformedFileCache(key string) io.ReadCloser { + f, meta, found := r.cache.getFromFile(key) + if !found { + return nil + } + r.transformedResourceMetadata = meta + r.sourceFilename = f.Name() + + return f +} + +func (r *transformedResource) Content() (interface{}, error) { + if err := r.initTransform(true); err != nil { + return nil, err + } + if err := r.initContent(); err != nil { + return "", err + } + return r.content, nil +} + +func (r *transformedResource) Data() interface{} { + return r.MetaData +} + +func (r *transformedResource) MediaType() media.Type { + if err := r.initTransform(false); err != nil { + return media.Type{} + } + m, _ := r.cache.rs.MediaTypes.GetByType(r.MediaTypeV) + return m +} + +func (r *transformedResource) Permalink() string { + if err := r.initTransform(false); err != nil { + return "" + } + return r.linker.permalinkFor(r.Target) +} + +func (r *transformedResource) RelPermalink() string { + if err := r.initTransform(false); err != nil { + return "" + } + return r.linker.relPermalinkFor(r.Target) +} + +func (r *transformedResource) initContent() error { + var err error + r.contentInit.Do(func() { + var b []byte + b, err := afero.ReadFile(r.cache.rs.Resources.Fs, r.sourceFilename) + if err != nil { + return + } + r.content = string(b) + }) + return err +} + +func (r *transformedResource) transform(setContent bool) (err error) { + + openPublishFileForWriting := func(relTargetPath string) (io.WriteCloser, error) { + return openFileForWriting(r.cache.rs.PublishFs, r.linker.relTargetPathFor(relTargetPath)) + } + + // This can be the last resource in a chain. + // Rewind and create a processing chain. + var chain []Resource + current := r + for { + rr := current.Resource + chain = append(chain[:0], append([]Resource{rr}, chain[0:]...)...) + if tr, ok := rr.(*transformedResource); ok { + current = tr + } else { + break + } + } + + // Append the current transformer at the end + chain = append(chain, r) + + first := chain[0] + + contentrc, err := contentReadSeekerCloser(first) + if err != nil { + return err + } + defer contentrc.Close() + + // Files with a suffix will be stored in cache (both on disk and in memory) + // partitioned by their suffix. There will be other files below /other. + // This partition is also how we determine what to delete on server reloads. + var key, base string + for _, element := range chain { + switch v := element.(type) { + case *transformedResource: + key = key + "_" + v.transformation.Key().key() + case permalinker: + r.linker = v + p := v.relTargetPath() + if p == "" { + panic("target path needed for key creation") + } + partition := ResourceKeyPartition(p) + base = partition + "/" + p + default: + return fmt.Errorf("transformation not supported for type %T", element) + } + } + + key = r.cache.cleanKey(base + "_" + helpers.MD5String(key)) + + cached, found := r.cache.get(key) + if found { + r.transferTransformedValues(cached.(*transformedResource)) + return + } + + // Acquire a write lock for the named transformation. + r.cache.nlocker.Lock(key) + // Check the cache again. + cached, found = r.cache.get(key) + if found { + r.transferTransformedValues(cached.(*transformedResource)) + r.cache.nlocker.Unlock(key) + return + } + defer r.cache.nlocker.Unlock(key) + defer r.cache.set(key, r) + + b1 := bp.GetBuffer() + b2 := bp.GetBuffer() + defer bp.PutBuffer(b1) + defer bp.PutBuffer(b2) + + tctx := &ResourceTransformationCtx{ + Data: r.transformedResourceMetadata.MetaData, + OpenResourcePublisher: openPublishFileForWriting, + } + + tctx.InMediaType = first.MediaType() + tctx.OutMediaType = first.MediaType() + tctx.From = contentrc + tctx.To = b1 + + if r.linker != nil { + tctx.InPath = r.linker.targetPath() + tctx.SourcePath = tctx.InPath + } + + counter := 0 + + var transformedContentr io.Reader + + for _, element := range chain { + tr, ok := element.(*transformedResource) + if !ok { + continue + } + counter++ + if counter != 1 { + tctx.InMediaType = tctx.OutMediaType + } + if counter%2 == 0 { + tctx.From = b1 + b2.Reset() + tctx.To = b2 + } else { + if counter != 1 { + // The first reader is the file. + tctx.From = b2 + } + b1.Reset() + tctx.To = b1 + } + + if err := tr.transformation.Transform(tctx); err != nil { + if err == errors.FeatureNotAvailableErr { + // This transformation is not available in this + // Hugo installation (scss not compiled in, PostCSS not available etc.) + // If a prepared bundle for this transformation chain is available, use that. + f := r.tryTransformedFileCache(key) + if f == nil { + return fmt.Errorf("failed to transform %q (%s): %s", tctx.InPath, tctx.InMediaType.Type(), err) + } + transformedContentr = f + defer f.Close() + + // The reader above is all we need. + break + } + + // Abort. + return err + } + + if tctx.OutPath != "" { + tctx.InPath = tctx.OutPath + tctx.OutPath = "" + } + } + + if transformedContentr == nil { + r.Target = tctx.InPath + r.MediaTypeV = tctx.OutMediaType.Type() + } + + publicw, err := openPublishFileForWriting(r.Target) + if err != nil { + r.transformErr = err + return + } + defer publicw.Close() + + publishwriters := []io.Writer{publicw} + + if transformedContentr == nil { + // Also write it to the cache + metaw, err := r.cache.writeMeta(key, r.transformedResourceMetadata) + if err != nil { + return err + } + r.sourceFilename = metaw.Name() + defer metaw.Close() + + publishwriters = append(publishwriters, metaw) + + if counter > 0 { + transformedContentr = tctx.To.(*bytes.Buffer) + } else { + transformedContentr = contentrc + } + } + + // Also write it to memory + var contentmemw *bytes.Buffer + + if setContent { + contentmemw = bp.GetBuffer() + defer bp.PutBuffer(contentmemw) + publishwriters = append(publishwriters, contentmemw) + } + + publishw := io.MultiWriter(publishwriters...) + _, r.transformErr = io.Copy(publishw, transformedContentr) + + if setContent { + r.contentInit.Do(func() { + r.content = contentmemw.String() + }) + } + + return nil + +} +func (r *transformedResource) initTransform(setContent bool) error { + r.transformInit.Do(func() { + if err := r.transform(setContent); err != nil { + r.transformErr = err + r.cache.rs.Logger.ERROR.Println("error: failed to transform resource:", err) + } + }) + return r.transformErr +} + +// contentReadSeekerCloser returns a ReadSeekerCloser if possible for a given Resource. +func contentReadSeekerCloser(r Resource) (ReadSeekCloser, error) { + switch rr := r.(type) { + case ReadSeekCloserResource: + rc, err := rr.ReadSeekCloser() + if err != nil { + return nil, err + } + return rc, nil + default: + return nil, fmt.Errorf("cannot tranform content of Resource of type %T", r) + + } +} diff --git a/resource/transform_test.go b/resource/transform_test.go new file mode 100644 index 000000000..df68e780d --- /dev/null +++ b/resource/transform_test.go @@ -0,0 +1,36 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +type testStruct struct { + Name string + V1 int64 + V2 int32 + V3 int + V4 uint64 +} + +func TestResourceTransformationKey(t *testing.T) { + // We really need this key to be portable across OSes. + key := NewResourceTransformationKey("testing", + testStruct{Name: "test", V1: int64(10), V2: int32(20), V3: 30, V4: uint64(40)}) + assert := require.New(t) + assert.Equal(key.key(), "testing_518996646957295636") +} diff --git a/source/filesystem_test.go b/source/filesystem_test.go index ee86c1487..2c1eeb171 100644 --- a/source/filesystem_test.go +++ b/source/filesystem_test.go @@ -75,12 +75,18 @@ func newTestConfig() *viper.Viper { v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") v.Set("archetypeDir", "archetypes") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") + v.Set("assetDir", "assets") return v } func newTestSourceSpec() *SourceSpec { v := newTestConfig() fs := hugofs.NewMem(v) - ps, _ := helpers.NewPathSpec(fs, v) + ps, err := helpers.NewPathSpec(fs, v) + if err != nil { + panic(err) + } return NewSourceSpec(ps, fs.Source) } diff --git a/tpl/collections/apply_test.go b/tpl/collections/apply_test.go index de24b06c8..0878844b2 100644 --- a/tpl/collections/apply_test.go +++ b/tpl/collections/apply_test.go @@ -25,8 +25,8 @@ import ( type templateFinder int -func (templateFinder) Lookup(name string) *tpl.TemplateAdapter { - return nil +func (templateFinder) Lookup(name string) (tpl.Template, bool) { + return nil, false } func (templateFinder) GetFuncs() map[string]interface{} { diff --git a/tpl/os/init.go b/tpl/os/init.go index 012f43b1f..3ef8702d6 100644 --- a/tpl/os/init.go +++ b/tpl/os/init.go @@ -37,14 +37,14 @@ func init() { ns.AddMethodMapping(ctx.ReadDir, []string{"readDir"}, [][2]string{ - {`{{ range (readDir ".") }}{{ .Name }}{{ end }}`, "README.txt"}, + {`{{ range (readDir "files") }}{{ .Name }}{{ end }}`, "README.txt"}, }, ) ns.AddMethodMapping(ctx.ReadFile, []string{"readFile"}, [][2]string{ - {`{{ readFile "README.txt" }}`, `Hugo Rocks!`}, + {`{{ readFile "files/README.txt" }}`, `Hugo Rocks!`}, }, ) diff --git a/tpl/os/os.go b/tpl/os/os.go index f7f9537ff..79d035d7e 100644 --- a/tpl/os/os.go +++ b/tpl/os/os.go @@ -34,7 +34,7 @@ func New(deps *deps.Deps) *Namespace { if deps.Fs != nil { rfs = deps.Fs.WorkingDir if deps.PathSpec != nil && deps.PathSpec.BaseFs != nil { - rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(deps.PathSpec.BaseFs.ContentFs, deps.Fs.WorkingDir)) + rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(deps.PathSpec.BaseFs.Content.Fs, deps.Fs.WorkingDir)) } } diff --git a/tpl/partials/partials.go b/tpl/partials/partials.go index beb09f426..18b8d7ed6 100644 --- a/tpl/partials/partials.go +++ b/tpl/partials/partials.go @@ -63,12 +63,13 @@ func (ns *Namespace) Include(name string, contextList ...interface{}) (interface } for _, n := range []string{"partials/" + name, "theme/partials/" + name} { - templ := ns.deps.Tmpl.Lookup(n) - if templ == nil { + templ, found := ns.deps.Tmpl.Lookup(n) + + if !found { // For legacy reasons. - templ = ns.deps.Tmpl.Lookup(n + ".html") + templ, found = ns.deps.Tmpl.Lookup(n + ".html") } - if templ != nil { + if found { b := bp.GetBuffer() defer bp.PutBuffer(b) @@ -76,7 +77,7 @@ func (ns *Namespace) Include(name string, contextList ...interface{}) (interface return "", err } - if _, ok := templ.Template.(*texttemplate.Template); ok { + if _, ok := templ.(*texttemplate.Template); ok { s := b.String() if ns.deps.Metrics != nil { ns.deps.Metrics.TrackValue(n, s) diff --git a/tpl/resources/init.go b/tpl/resources/init.go new file mode 100644 index 000000000..3e750f325 --- /dev/null +++ b/tpl/resources/init.go @@ -0,0 +1,68 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/tpl/internal" +) + +const name = "resources" + +func init() { + f := func(d *deps.Deps) *internal.TemplateFuncsNamespace { + ctx, err := New(d) + if err != nil { + // TODO(bep) no panic. + panic(err) + } + + ns := &internal.TemplateFuncsNamespace{ + Name: name, + Context: func(args ...interface{}) interface{} { return ctx }, + } + + ns.AddMethodMapping(ctx.Get, + nil, + [][2]string{}, + ) + + // Add aliases for the most common transformations. + + ns.AddMethodMapping(ctx.Fingerprint, + []string{"fingerprint"}, + [][2]string{}, + ) + + ns.AddMethodMapping(ctx.Minify, + []string{"minify"}, + [][2]string{}, + ) + + ns.AddMethodMapping(ctx.ToCSS, + []string{"toCSS"}, + [][2]string{}, + ) + + ns.AddMethodMapping(ctx.PostCSS, + []string{"postCSS"}, + [][2]string{}, + ) + + return ns + + } + + internal.AddTemplateFuncsNamespace(f) +} diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go new file mode 100644 index 000000000..5d4f6e315 --- /dev/null +++ b/tpl/resources/resources.go @@ -0,0 +1,255 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/resource" + "github.com/gohugoio/hugo/resource/bundler" + "github.com/gohugoio/hugo/resource/create" + "github.com/gohugoio/hugo/resource/integrity" + "github.com/gohugoio/hugo/resource/minifiers" + "github.com/gohugoio/hugo/resource/postcss" + "github.com/gohugoio/hugo/resource/templates" + "github.com/gohugoio/hugo/resource/tocss/scss" + "github.com/spf13/cast" +) + +// New returns a new instance of the resources-namespaced template functions. +func New(deps *deps.Deps) (*Namespace, error) { + scssClient, err := scss.New(deps.BaseFs.Assets, deps.ResourceSpec) + if err != nil { + return nil, err + } + return &Namespace{ + deps: deps, + scssClient: scssClient, + createClient: create.New(deps.ResourceSpec), + bundlerClient: bundler.New(deps.ResourceSpec), + integrityClient: integrity.New(deps.ResourceSpec), + minifyClient: minifiers.New(deps.ResourceSpec), + postcssClient: postcss.New(deps.ResourceSpec), + templatesClient: templates.New(deps.ResourceSpec, deps.TextTmpl), + }, nil +} + +// Namespace provides template functions for the "resources" namespace. +type Namespace struct { + deps *deps.Deps + + createClient *create.Client + bundlerClient *bundler.Client + scssClient *scss.Client + integrityClient *integrity.Client + minifyClient *minifiers.Client + postcssClient *postcss.Client + templatesClient *templates.Client +} + +// Get locates the filename given in Hugo's filesystems: static, assets and content (in that order) +// and creates a Resource object that can be used for further transformations. +func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) { + filenamestr, err := cast.ToStringE(filename) + if err != nil { + return nil, err + } + + filenamestr = filepath.Clean(filenamestr) + + // Resource Get'ing is currently limited to /assets to make it simpler + // to control the behaviour of publishing and partial rebuilding. + return ns.createClient.Get(ns.deps.BaseFs.Assets.Fs, filenamestr) + +} + +// Concat concatenates a slice of Resource objects. These resources must +// (currently) be of the same Media Type. +func (ns *Namespace) Concat(targetPathIn interface{}, r []interface{}) (resource.Resource, error) { + targetPath, err := cast.ToStringE(targetPathIn) + if err != nil { + return nil, err + } + rr := make([]resource.Resource, len(r)) + for i := 0; i < len(r); i++ { + rv, ok := r[i].(resource.Resource) + if !ok { + return nil, fmt.Errorf("cannot concat type %T", rv) + } + rr[i] = rv + } + return ns.bundlerClient.Concat(targetPath, rr) +} + +// FromString creates a Resource from a string published to the relative target path. +func (ns *Namespace) FromString(targetPathIn, contentIn interface{}) (resource.Resource, error) { + targetPath, err := cast.ToStringE(targetPathIn) + if err != nil { + return nil, err + } + content, err := cast.ToStringE(contentIn) + if err != nil { + return nil, err + } + + return ns.createClient.FromString(targetPath, content) +} + +// ExecuteAsTemplate creates a Resource from a Go template, parsed and executed with +// the given data, and published to the relative target path. +func (ns *Namespace) ExecuteAsTemplate(args ...interface{}) (resource.Resource, error) { + if len(args) != 3 { + return nil, fmt.Errorf("must provide targetPath, the template data context and a Resource object") + } + targetPath, err := cast.ToStringE(args[0]) + if err != nil { + return nil, err + } + data := args[1] + + r, ok := args[2].(resource.Resource) + if !ok { + return nil, fmt.Errorf("type %T not supported in Resource transformations", args[2]) + } + + return ns.templatesClient.ExecuteAsTemplate(r, targetPath, data) +} + +// Fingerprint transforms the given Resource with a MD5 hash of the content in +// the RelPermalink and Permalink. +func (ns *Namespace) Fingerprint(args ...interface{}) (resource.Resource, error) { + if len(args) < 1 || len(args) > 2 { + return nil, errors.New("must provide a Resource and (optional) crypto algo") + } + + var algo string + resIdx := 0 + + if len(args) == 2 { + resIdx = 1 + var err error + algo, err = cast.ToStringE(args[0]) + if err != nil { + return nil, err + } + } + + r, ok := args[resIdx].(resource.Resource) + if !ok { + return nil, fmt.Errorf("%T is not a Resource", args[resIdx]) + } + + return ns.integrityClient.Fingerprint(r, algo) +} + +// Minify minifies the given Resource using the MediaType to pick the correct +// minifier. +func (ns *Namespace) Minify(r resource.Resource) (resource.Resource, error) { + return ns.minifyClient.Minify(r) +} + +// ToCSS converts the given Resource to CSS. You can optional provide an Options +// object or a target path (string) as first argument. +func (ns *Namespace) ToCSS(args ...interface{}) (resource.Resource, error) { + var ( + r resource.Resource + m map[string]interface{} + targetPath string + err error + ok bool + ) + + r, targetPath, ok = ns.resolveIfFirstArgIsString(args) + + if !ok { + r, m, err = ns.resolveArgs(args) + if err != nil { + return nil, err + } + } + + var options scss.Options + if targetPath != "" { + options.TargetPath = targetPath + } else if m != nil { + options, err = scss.DecodeOptions(m) + if err != nil { + return nil, err + } + } + + return ns.scssClient.ToCSS(r, options) +} + +// PostCSS processes the given Resource with PostCSS +func (ns *Namespace) PostCSS(args ...interface{}) (resource.Resource, error) { + r, m, err := ns.resolveArgs(args) + if err != nil { + return nil, err + } + var options postcss.Options + if m != nil { + options, err = postcss.DecodeOptions(m) + if err != nil { + return nil, err + } + } + + return ns.postcssClient.Process(r, options) +} + +// We allow string or a map as the first argument in some cases. +func (ns *Namespace) resolveIfFirstArgIsString(args []interface{}) (resource.Resource, string, bool) { + if len(args) != 2 { + return nil, "", false + } + + v1, ok1 := args[0].(string) + if !ok1 { + return nil, "", false + } + v2, ok2 := args[1].(resource.Resource) + + return v2, v1, ok2 +} + +// This roundabout way of doing it is needed to get both pipeline behaviour and options as arguments. +func (ns *Namespace) resolveArgs(args []interface{}) (resource.Resource, map[string]interface{}, error) { + if len(args) == 0 { + return nil, nil, errors.New("no Resource provided in transformation") + } + + if len(args) == 1 { + r, ok := args[0].(resource.Resource) + if !ok { + return nil, nil, fmt.Errorf("type %T not supported in Resource transformations", args[0]) + } + return r, nil, nil + } + + r, ok := args[1].(resource.Resource) + if !ok { + return nil, nil, fmt.Errorf("type %T not supported in Resource transformations", args[0]) + } + + m, err := cast.ToStringMapE(args[0]) + if err != nil { + return nil, nil, fmt.Errorf("invalid options type: %s", err) + } + + return r, m, nil +} diff --git a/tpl/template.go b/tpl/template.go index e04d2cc6c..2cef92bb2 100644 --- a/tpl/template.go +++ b/tpl/template.go @@ -38,13 +38,15 @@ type TemplateHandler interface { LoadTemplates(prefix string) PrintErrors() + NewTextTemplate() TemplateParseFinder + MarkReady() RebuildClone() } // TemplateFinder finds templates. type TemplateFinder interface { - Lookup(name string) *TemplateAdapter + Lookup(name string) (Template, bool) } // Template is the common interface between text/template and html/template. @@ -53,6 +55,17 @@ type Template interface { Name() string } +// TemplateParser is used to parse ad-hoc templates, e.g. in the Resource chain. +type TemplateParser interface { + Parse(name, tpl string) (Template, error) +} + +// TemplateParseFinder provides both parsing and finding. +type TemplateParseFinder interface { + TemplateParser + TemplateFinder +} + // TemplateExecutor adds some extras to Template. type TemplateExecutor interface { Template diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index e838ebc57..f19c312ec 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -55,7 +55,7 @@ var ( _ templateFuncsterTemplater = (*textTemplates)(nil) ) -// Protecting global map access (Amber) +// Protecting global map access (Amber) var amberMu sync.Mutex type templateErr struct { @@ -70,18 +70,26 @@ type templateLoader interface { } type templateFuncsterTemplater interface { + templateFuncsterSetter tpl.TemplateFinder setFuncs(funcMap map[string]interface{}) +} + +type templateFuncsterSetter interface { setTemplateFuncster(f *templateFuncster) } // templateHandler holds the templates in play. // It implements the templateLoader and tpl.TemplateHandler interfaces. type templateHandler struct { + mu sync.Mutex + // text holds all the pure text templates. text *textTemplates html *htmlTemplates + extTextTemplates []*textTemplate + amberFuncMap template.FuncMap errors []*templateErr @@ -93,6 +101,19 @@ type templateHandler struct { *deps.Deps } +// NewTextTemplate provides a text template parser that has all the Hugo +// template funcs etc. built-in. +func (t *templateHandler) NewTextTemplate() tpl.TemplateParseFinder { + t.mu.Lock() + t.mu.Unlock() + + tt := &textTemplate{t: texttemplate.New("")} + t.extTextTemplates = append(t.extTextTemplates, tt) + + return tt + +} + func (t *templateHandler) addError(name string, err error) { t.errors = append(t.errors, &templateErr{name, err}) } @@ -111,7 +132,7 @@ func (t *templateHandler) PrintErrors() { // Lookup tries to find a template with the given name in both template // collections: First HTML, then the plain text template collection. -func (t *templateHandler) Lookup(name string) *tpl.TemplateAdapter { +func (t *templateHandler) Lookup(name string) (tpl.Template, bool) { if strings.HasPrefix(name, textTmplNamePrefix) { // The caller has explicitly asked for a text template, so only look @@ -123,8 +144,8 @@ func (t *templateHandler) Lookup(name string) *tpl.TemplateAdapter { } // Look in both - if te := t.html.Lookup(name); te != nil { - return te + if te, found := t.html.Lookup(name); found { + return te, true } return t.text.Lookup(name) @@ -136,7 +157,7 @@ func (t *templateHandler) clone(d *deps.Deps) *templateHandler { Deps: d, layoutsFs: d.BaseFs.Layouts.Fs, html: &htmlTemplates{t: template.Must(t.html.t.Clone()), overlays: make(map[string]*template.Template)}, - text: &textTemplates{t: texttemplate.Must(t.text.t.Clone()), overlays: make(map[string]*texttemplate.Template)}, + text: &textTemplates{textTemplate: &textTemplate{t: texttemplate.Must(t.text.t.Clone())}, overlays: make(map[string]*texttemplate.Template)}, errors: make([]*templateErr, 0), } @@ -171,8 +192,8 @@ func newTemplateAdapter(deps *deps.Deps) *templateHandler { overlays: make(map[string]*template.Template), } textT := &textTemplates{ - t: texttemplate.New(""), - overlays: make(map[string]*texttemplate.Template), + textTemplate: &textTemplate{t: texttemplate.New("")}, + overlays: make(map[string]*texttemplate.Template), } return &templateHandler{ Deps: deps, @@ -205,12 +226,12 @@ func (t *htmlTemplates) setTemplateFuncster(f *templateFuncster) { t.funcster = f } -func (t *htmlTemplates) Lookup(name string) *tpl.TemplateAdapter { +func (t *htmlTemplates) Lookup(name string) (tpl.Template, bool) { templ := t.lookup(name) if templ == nil { - return nil + return nil, false } - return &tpl.TemplateAdapter{Template: templ, Metrics: t.funcster.Deps.Metrics} + return &tpl.TemplateAdapter{Template: templ, Metrics: t.funcster.Deps.Metrics}, true } func (t *htmlTemplates) lookup(name string) *template.Template { @@ -233,27 +254,25 @@ func (t *htmlTemplates) lookup(name string) *template.Template { return nil } -type textTemplates struct { - funcster *templateFuncster - - t *texttemplate.Template +func (t *textTemplates) setTemplateFuncster(f *templateFuncster) { + t.funcster = f +} +type textTemplates struct { + *textTemplate + funcster *templateFuncster clone *texttemplate.Template cloneClone *texttemplate.Template overlays map[string]*texttemplate.Template } -func (t *textTemplates) setTemplateFuncster(f *templateFuncster) { - t.funcster = f -} - -func (t *textTemplates) Lookup(name string) *tpl.TemplateAdapter { +func (t *textTemplates) Lookup(name string) (tpl.Template, bool) { templ := t.lookup(name) if templ == nil { - return nil + return nil, false } - return &tpl.TemplateAdapter{Template: templ, Metrics: t.funcster.Deps.Metrics} + return &tpl.TemplateAdapter{Template: templ, Metrics: t.funcster.Deps.Metrics}, true } func (t *textTemplates) lookup(name string) *texttemplate.Template { @@ -336,9 +355,34 @@ func (t *htmlTemplates) addLateTemplate(name, tpl string) error { return t.addTemplateIn(t.clone, name, tpl) } +type textTemplate struct { + t *texttemplate.Template +} + +func (t *textTemplate) Parse(name, tpl string) (tpl.Template, error) { + return t.parSeIn(t.t, name, tpl) +} + +func (t *textTemplate) Lookup(name string) (tpl.Template, bool) { + tpl := t.t.Lookup(name) + return tpl, tpl != nil +} + +func (t *textTemplate) parSeIn(tt *texttemplate.Template, name, tpl string) (*texttemplate.Template, error) { + templ, err := tt.New(name).Parse(tpl) + if err != nil { + return nil, err + } + + if err := applyTemplateTransformersToTextTemplate(templ); err != nil { + return nil, err + } + return templ, nil +} + func (t *textTemplates) addTemplateIn(tt *texttemplate.Template, name, tpl string) error { name = strings.TrimPrefix(name, textTmplNamePrefix) - templ, err := tt.New(name).Parse(tpl) + templ, err := t.parSeIn(tt, name, tpl) if err != nil { return err } @@ -467,17 +511,22 @@ func (t *templateHandler) initFuncs() { // Both template types will get their own funcster instance, which // in the current case contains the same set of funcs. - for _, funcsterHolder := range []templateFuncsterTemplater{t.html, t.text} { + funcMap := createFuncMap(t.Deps) + for _, funcsterHolder := range []templateFuncsterSetter{t.html, t.text} { funcster := newTemplateFuncster(t.Deps) // The URL funcs in the funcMap is somewhat language dependent, // so we need to wait until the language and site config is loaded. - funcster.initFuncMap() + funcster.initFuncMap(funcMap) funcsterHolder.setTemplateFuncster(funcster) } + for _, extText := range t.extTextTemplates { + extText.t.Funcs(funcMap) + } + // Amber is HTML only. t.amberFuncMap = template.FuncMap{} diff --git a/tpl/tplimpl/templateFuncster.go b/tpl/tplimpl/templateFuncster.go index e6bbde8ec..9490123ab 100644 --- a/tpl/tplimpl/templateFuncster.go +++ b/tpl/tplimpl/templateFuncster.go @@ -51,12 +51,12 @@ func (t *templateFuncster) partial(name string, contextList ...interface{}) (int } for _, n := range []string{"partials/" + name, "theme/partials/" + name} { - templ := t.Tmpl.Lookup(n) - if templ == nil { + templ, found := t.Tmpl.Lookup(n) + if !found { // For legacy reasons. - templ = t.Tmpl.Lookup(n + ".html") + templ, found = t.Tmpl.Lookup(n + ".html") } - if templ != nil { + if found { b := bp.GetBuffer() defer bp.PutBuffer(b) @@ -64,7 +64,7 @@ func (t *templateFuncster) partial(name string, contextList ...interface{}) (int return "", err } - if _, ok := templ.Template.(*texttemplate.Template); ok { + if _, ok := templ.(*texttemplate.Template); ok { return b.String(), nil } diff --git a/tpl/tplimpl/templateProvider.go b/tpl/tplimpl/templateProvider.go index af89fed11..df44e81a6 100644 --- a/tpl/tplimpl/templateProvider.go +++ b/tpl/tplimpl/templateProvider.go @@ -30,6 +30,8 @@ func (*TemplateProvider) Update(deps *deps.Deps) error { newTmpl := newTemplateAdapter(deps) deps.Tmpl = newTmpl + deps.TextTmpl = newTmpl.NewTextTemplate() + newTmpl.initFuncs() newTmpl.loadEmbedded() diff --git a/tpl/tplimpl/template_funcs.go b/tpl/tplimpl/template_funcs.go index 6ce387aca..f1ed7f36f 100644 --- a/tpl/tplimpl/template_funcs.go +++ b/tpl/tplimpl/template_funcs.go @@ -18,6 +18,8 @@ package tplimpl import ( "html/template" + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/tpl/internal" // Init the namespaces @@ -35,6 +37,7 @@ import ( _ "github.com/gohugoio/hugo/tpl/os" _ "github.com/gohugoio/hugo/tpl/partials" _ "github.com/gohugoio/hugo/tpl/path" + _ "github.com/gohugoio/hugo/tpl/resources" _ "github.com/gohugoio/hugo/tpl/safe" _ "github.com/gohugoio/hugo/tpl/strings" _ "github.com/gohugoio/hugo/tpl/time" @@ -42,12 +45,12 @@ import ( _ "github.com/gohugoio/hugo/tpl/urls" ) -func (t *templateFuncster) initFuncMap() { +func createFuncMap(d *deps.Deps) map[string]interface{} { funcMap := template.FuncMap{} // Merge the namespace funcs for _, nsf := range internal.TemplateFuncsNamespaceRegistry { - ns := nsf(t.Deps) + ns := nsf(d) if _, exists := funcMap[ns.Name]; exists { panic(ns.Name + " is a duplicate template func") } @@ -61,8 +64,13 @@ func (t *templateFuncster) initFuncMap() { } } + } + return funcMap + +} +func (t *templateFuncster) initFuncMap(funcMap template.FuncMap) { t.funcMap = funcMap t.Tmpl.(*templateHandler).setFuncs(funcMap) } diff --git a/tpl/tplimpl/template_funcs_test.go b/tpl/tplimpl/template_funcs_test.go index a1745282d..341be805a 100644 --- a/tpl/tplimpl/template_funcs_test.go +++ b/tpl/tplimpl/template_funcs_test.go @@ -51,6 +51,9 @@ func newTestConfig() config.Provider { v.Set("i18nDir", "i18n") v.Set("layoutDir", "layouts") v.Set("archetypeDir", "archetypes") + v.Set("assetDir", "assets") + v.Set("resourceDir", "resources") + v.Set("publishDir", "public") return v } @@ -76,12 +79,13 @@ func TestTemplateFuncsExamples(t *testing.T) { v.Set("workingDir", workingDir) v.Set("multilingual", true) v.Set("contentDir", "content") + v.Set("assetDir", "assets") v.Set("baseURL", "http://mysite.com/hugo/") v.Set("CurrentContentLanguage", langs.NewLanguage("en", v)) fs := hugofs.NewMem(v) - afero.WriteFile(fs.Source, filepath.Join(workingDir, "README.txt"), []byte("Hugo Rocks!"), 0755) + afero.WriteFile(fs.Source, filepath.Join(workingDir, "files", "README.txt"), []byte("Hugo Rocks!"), 0755) depsCfg := newDepsConfig(v) depsCfg.Fs = fs @@ -113,7 +117,8 @@ func TestTemplateFuncsExamples(t *testing.T) { require.NoError(t, d.LoadResources()) var b bytes.Buffer - require.NoError(t, d.Tmpl.Lookup("test").Execute(&b, &data)) + templ, _ := d.Tmpl.Lookup("test") + require.NoError(t, templ.Execute(&b, &data)) if b.String() != expected { t.Fatalf("%s[%d]: got %q expected %q", ns.Name, i, b.String(), expected) } diff --git a/tpl/tplimpl/template_test.go b/tpl/tplimpl/template_test.go index 3ce2a88a2..683850fa5 100644 --- a/tpl/tplimpl/template_test.go +++ b/tpl/tplimpl/template_test.go @@ -18,6 +18,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/tpl" "github.com/stretchr/testify/require" ) @@ -43,20 +44,22 @@ func TestHTMLEscape(t *testing.T) { d, err := deps.New(depsCfg) assert.NoError(err) - tpl := `{{ "<h1>Hi!</h1>" | safeHTML }}` + templ := `{{ "<h1>Hi!</h1>" | safeHTML }}` provider := DefaultTemplateProvider provider.Update(d) h := d.Tmpl.(handler) - assert.NoError(h.addTemplate("shortcodes/myShort.html", tpl)) + assert.NoError(h.addTemplate("shortcodes/myShort.html", templ)) - s, err := d.Tmpl.Lookup("shortcodes/myShort.html").ExecuteToString(data) + tt, _ := d.Tmpl.Lookup("shortcodes/myShort.html") + s, err := tt.(tpl.TemplateExecutor).ExecuteToString(data) assert.NoError(err) assert.Contains(s, "<h1>Hi!</h1>") - s, err = d.Tmpl.Lookup("shortcodes/myShort").ExecuteToString(data) + tt, _ = d.Tmpl.Lookup("shortcodes/myShort") + s, err = tt.(tpl.TemplateExecutor).ExecuteToString(data) assert.NoError(err) assert.Contains(s, "<h1>Hi!</h1>") |