diff options
Diffstat (limited to 'helpers')
-rw-r--r-- | helpers/content.go | 11 | ||||
-rw-r--r-- | helpers/content_test.go | 6 | ||||
-rw-r--r-- | helpers/general.go | 32 | ||||
-rw-r--r-- | helpers/general_test.go | 28 | ||||
-rw-r--r-- | helpers/path.go | 105 | ||||
-rw-r--r-- | helpers/path_test.go | 5 | ||||
-rw-r--r-- | helpers/pathspec.go | 6 | ||||
-rw-r--r-- | helpers/processing_stats.go | 2 | ||||
-rw-r--r-- | helpers/url.go | 47 | ||||
-rw-r--r-- | helpers/url_test.go | 37 |
10 files changed, 64 insertions, 215 deletions
diff --git a/helpers/content.go b/helpers/content.go index a3abb334d..889294382 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -30,7 +30,6 @@ import ( "github.com/spf13/afero" "github.com/gohugoio/hugo/markup/converter" - "github.com/gohugoio/hugo/markup/converter/hooks" "github.com/gohugoio/hugo/markup" @@ -38,19 +37,15 @@ import ( ) var ( - openingPTag = []byte("<p>") - closingPTag = []byte("</p>") - paragraphIndicator = []byte("<p") - closingIndicator = []byte("</") + openingPTag = []byte("<p>") + closingPTag = []byte("</p>") ) // ContentSpec provides functionality to render markdown content. type ContentSpec struct { Converters markup.ConverterProvider anchorNameSanitizer converter.AnchorNameSanitizer - getRenderer func(t hooks.RendererType, id any) any - - Cfg config.AllProvider + Cfg config.AllProvider } // NewContentSpec returns a ContentSpec initialized diff --git a/helpers/content_test.go b/helpers/content_test.go index 72e3eeb49..e2bf501d2 100644 --- a/helpers/content_test.go +++ b/helpers/content_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,8 +24,6 @@ import ( "github.com/gohugoio/hugo/helpers" ) -const tstHTMLContent = "<!DOCTYPE html><html><head><script src=\"http://two/foobar.js\"></script></head><body><nav><ul><li hugo-nav=\"section_0\"></li><li hugo-nav=\"section_1\"></li></ul></nav><article>content <a href=\"http://two/foobar\">foobar</a>. Follow up</article><p>This is some text.<br>And some more.</p></body></html>" - func TestTrimShortHTML(t *testing.T) { tests := []struct { input, output []byte @@ -68,7 +66,6 @@ func BenchmarkTestTruncateWordsToWholeSentence(b *testing.B) { } func TestTruncateWordsToWholeSentence(t *testing.T) { - type test struct { input, expected string max int @@ -101,7 +98,6 @@ func TestTruncateWordsToWholeSentence(t *testing.T) { } func TestTruncateWordsByRune(t *testing.T) { - type test struct { input, expected string max int diff --git a/helpers/general.go b/helpers/general.go index b16aec0b0..35e35a7e0 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -196,6 +196,7 @@ func ReaderContains(r io.Reader, subslice []byte) bool { func GetTitleFunc(style string) func(s string) string { switch strings.ToLower(style) { case "go": + //lint:ignore SA1019 keep for now. return strings.Title case "chicago": tc := transform.NewTitleConverter(transform.ChicagoStyle) @@ -263,10 +264,11 @@ func MD5String(f string) string { return hex.EncodeToString(h.Sum([]byte{})) } -// MD5FromFileFast creates a MD5 hash from the given file. It only reads parts of +// MD5FromReaderFast creates a MD5 hash from the given file. It only reads parts of // the file for speed, so don't use it if the files are very subtly different. // It will not close the file. -func MD5FromFileFast(r io.ReadSeeker) (string, error) { +// It will return the MD5 hash and the size of r in bytes. +func MD5FromReaderFast(r io.ReadSeeker) (string, int64, error) { const ( // Do not change once set in stone! maxChunks = 8 @@ -284,7 +286,7 @@ func MD5FromFileFast(r io.ReadSeeker) (string, error) { if err == io.EOF { break } - return "", err + return "", 0, err } } @@ -294,12 +296,14 @@ func MD5FromFileFast(r io.ReadSeeker) (string, error) { h.Write(buff) break } - return "", err + return "", 0, err } h.Write(buff) } - return hex.EncodeToString(h.Sum(nil)), nil + size, _ := r.Seek(0, io.SeekEnd) + + return hex.EncodeToString(h.Sum(nil)), size, nil } // MD5FromReader creates a MD5 hash from the given reader. @@ -328,3 +332,21 @@ func PrintFs(fs afero.Fs, path string, w io.Writer) { return nil }) } + +// FormatByteCount pretty formats b. +func FormatByteCount(bc uint64) string { + const ( + Gigabyte = 1 << 30 + Megabyte = 1 << 20 + Kilobyte = 1 << 10 + ) + switch { + case bc > Gigabyte || -bc > Gigabyte: + return fmt.Sprintf("%.2f GB", float64(bc)/Gigabyte) + case bc > Megabyte || -bc > Megabyte: + return fmt.Sprintf("%.2f MB", float64(bc)/Megabyte) + case bc > Kilobyte || -bc > Kilobyte: + return fmt.Sprintf("%.2f KB", float64(bc)/Kilobyte) + } + return fmt.Sprintf("%d B", bc) +} diff --git a/helpers/general_test.go b/helpers/general_test.go index 1463458fa..54607d699 100644 --- a/helpers/general_test.go +++ b/helpers/general_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -259,19 +259,19 @@ func TestUniqueStringsSorted(t *testing.T) { func TestFastMD5FromFile(t *testing.T) { fs := afero.NewMemMapFs() - if err := afero.WriteFile(fs, "small.txt", []byte("abc"), 0777); err != nil { + if err := afero.WriteFile(fs, "small.txt", []byte("abc"), 0o777); err != nil { t.Fatal(err) } - if err := afero.WriteFile(fs, "small2.txt", []byte("abd"), 0777); err != nil { + if err := afero.WriteFile(fs, "small2.txt", []byte("abd"), 0o777); err != nil { t.Fatal(err) } - if err := afero.WriteFile(fs, "bigger.txt", []byte(strings.Repeat("a bc d e", 100)), 0777); err != nil { + if err := afero.WriteFile(fs, "bigger.txt", []byte(strings.Repeat("a bc d e", 100)), 0o777); err != nil { t.Fatal(err) } - if err := afero.WriteFile(fs, "bigger2.txt", []byte(strings.Repeat("c d e f g", 100)), 0777); err != nil { + if err := afero.WriteFile(fs, "bigger2.txt", []byte(strings.Repeat("c d e f g", 100)), 0o777); err != nil { t.Fatal(err) } @@ -292,19 +292,19 @@ func TestFastMD5FromFile(t *testing.T) { defer bf1.Close() defer bf2.Close() - m1, err := helpers.MD5FromFileFast(sf1) + m1, _, err := helpers.MD5FromReaderFast(sf1) c.Assert(err, qt.IsNil) c.Assert(m1, qt.Equals, "e9c8989b64b71a88b4efb66ad05eea96") - m2, err := helpers.MD5FromFileFast(sf2) + m2, _, err := helpers.MD5FromReaderFast(sf2) c.Assert(err, qt.IsNil) c.Assert(m2, qt.Not(qt.Equals), m1) - m3, err := helpers.MD5FromFileFast(bf1) + m3, _, err := helpers.MD5FromReaderFast(bf1) c.Assert(err, qt.IsNil) c.Assert(m3, qt.Not(qt.Equals), m2) - m4, err := helpers.MD5FromFileFast(bf2) + m4, _, err := helpers.MD5FromReaderFast(bf2) c.Assert(err, qt.IsNil) c.Assert(m4, qt.Not(qt.Equals), m3) @@ -320,7 +320,7 @@ func BenchmarkMD5FromFileFast(b *testing.B) { b.Run(fmt.Sprintf("full=%t", full), func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - if err := afero.WriteFile(fs, "file.txt", []byte(strings.Repeat("1234567890", 2000)), 0777); err != nil { + if err := afero.WriteFile(fs, "file.txt", []byte(strings.Repeat("1234567890", 2000)), 0o777); err != nil { b.Fatal(err) } f, err := fs.Open("file.txt") @@ -333,7 +333,7 @@ func BenchmarkMD5FromFileFast(b *testing.B) { b.Fatal(err) } } else { - if _, err := helpers.MD5FromFileFast(f); err != nil { + if _, _, err := helpers.MD5FromReaderFast(f); err != nil { b.Fatal(err) } } @@ -350,7 +350,7 @@ func BenchmarkUniqueStrings(b *testing.B) { for i := 0; i < b.N; i++ { result := helpers.UniqueStrings(input) if len(result) != 6 { - b.Fatal(fmt.Sprintf("invalid count: %d", len(result))) + b.Fatalf("invalid count: %d", len(result)) } } }) @@ -369,7 +369,7 @@ func BenchmarkUniqueStrings(b *testing.B) { result := helpers.UniqueStringsReuse(inputc) if len(result) != 6 { - b.Fatal(fmt.Sprintf("invalid count: %d", len(result))) + b.Fatalf("invalid count: %d", len(result)) } } }) @@ -388,7 +388,7 @@ func BenchmarkUniqueStrings(b *testing.B) { result := helpers.UniqueStringsSorted(inputc) if len(result) != 6 { - b.Fatal(fmt.Sprintf("invalid count: %d", len(result))) + b.Fatalf("invalid count: %d", len(result)) } } }) diff --git a/helpers/path.go b/helpers/path.go index 3172d3452..4a6c9a688 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,12 +23,12 @@ import ( "regexp" "sort" "strings" - "unicode" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/text" "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/common/hugio" @@ -41,7 +41,11 @@ import ( // whilst preserving the original casing of the string. // E.g. Social Media -> Social-Media func (p *PathSpec) MakePath(s string) string { - return p.UnicodeSanitize(s) + s = paths.Sanitize(s) + if p.Cfg.RemovePathAccents() { + s = text.RemoveAccentsString(s) + } + return s } // MakePathsSanitized applies MakePathSanitized on every item in the slice @@ -59,74 +63,13 @@ func (p *PathSpec) MakePathSanitized(s string) string { return strings.ToLower(p.MakePath(s)) } -// ToSlashTrimLeading is just a filepath.ToSlaas with an added / prefix trimmer. -func ToSlashTrimLeading(s string) string { - return strings.TrimPrefix(filepath.ToSlash(s), "/") -} - // MakeTitle converts the path given to a suitable title, trimming whitespace // and replacing hyphens with whitespace. func MakeTitle(inpath string) string { return strings.Replace(strings.TrimSpace(inpath), "-", " ", -1) } -// From https://golang.org/src/net/url/url.go -func ishex(c rune) bool { - switch { - case '0' <= c && c <= '9': - return true - case 'a' <= c && c <= 'f': - return true - case 'A' <= c && c <= 'F': - return true - } - return false -} - -// UnicodeSanitize sanitizes string to be used in Hugo URL's, allowing only -// a predefined set of special Unicode characters. -// If RemovePathAccents configuration flag is enabled, Unicode accents -// are also removed. -// Hyphens in the original input are maintained. -// Spaces will be replaced with a single hyphen, and sequential replacement hyphens will be reduced to one. -func (p *PathSpec) UnicodeSanitize(s string) string { - if p.Cfg.RemovePathAccents() { - s = text.RemoveAccentsString(s) - } - - source := []rune(s) - target := make([]rune, 0, len(source)) - var ( - prependHyphen bool - wasHyphen bool - ) - - for i, r := range source { - isAllowed := r == '.' || r == '/' || r == '\\' || r == '_' || r == '#' || r == '+' || r == '~' || r == '-' || r == '@' - isAllowed = isAllowed || unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r) - isAllowed = isAllowed || (r == '%' && i+2 < len(source) && ishex(source[i+1]) && ishex(source[i+2])) - - if isAllowed { - // track explicit hyphen in input; no need to add a new hyphen if - // we just saw one. - wasHyphen = r == '-' - - if prependHyphen { - // if currently have a hyphen, don't prepend an extra one - if !wasHyphen { - target = append(target, '-') - } - prependHyphen = false - } - target = append(target, r) - } else if len(target) > 0 && !wasHyphen && unicode.IsSpace(r) { - prependHyphen = true - } - } - - return string(target) -} - +// MakeTitleInPath converts the path given to a suitable title, trimming whitespace func MakePathRelative(inPath string, possibleDirectories ...string) (string, error) { for _, currentPath := range possibleDirectories { if strings.HasPrefix(inPath, currentPath) { @@ -317,13 +260,12 @@ func FindCWD() (string, error) { return path, nil } -// SymbolicWalk is like filepath.Walk, but it follows symbolic links. -func SymbolicWalk(fs afero.Fs, root string, walker hugofs.WalkFunc) error { +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. +func Walk(fs afero.Fs, root string, walker hugofs.WalkFunc) error { if _, isOs := fs.(*afero.OsFs); isOs { - // Mainly to track symlinks. fs = hugofs.NewBaseFileDecorator(fs) } - w := hugofs.NewWalkway(hugofs.WalkwayConfig{ Fs: fs, Root: root, @@ -333,16 +275,6 @@ func SymbolicWalk(fs afero.Fs, root string, walker hugofs.WalkFunc) error { return w.Walk() } -// LstatIfPossible can be used to call Lstat if possible, else Stat. -func LstatIfPossible(fs afero.Fs, path string) (os.FileInfo, error) { - if lstater, ok := fs.(afero.Lstater); ok { - fi, _, err := lstater.LstatIfPossible(path) - return fi, err - } - - return fs.Stat(path) -} - // SafeWriteToDisk is the same as WriteToDisk // but it also checks to see if file/directory already exists. func SafeWriteToDisk(inpath string, r io.Reader, fs afero.Fs) (err error) { @@ -382,7 +314,7 @@ func OpenFileForWriting(fs afero.Fs, filename string) (afero.File, error) { if !herrors.IsNotExist(err) { return nil, err } - if err = fs.MkdirAll(filepath.Dir(filename), 0777); err != nil { // before umask + if err = fs.MkdirAll(filepath.Dir(filename), 0o777); err != nil { // before umask return nil, err } f, err = fs.Create(filename) @@ -402,7 +334,7 @@ func GetCacheDir(fs afero.Fs, cacheDir string) (string, error) { return "", err } if !exists { - err := fs.MkdirAll(cacheDir, 0777) // Before umask + err := fs.MkdirAll(cacheDir, 0o777) // Before umask if err != nil { return "", fmt.Errorf("failed to create cache dir: %w", err) } @@ -417,7 +349,7 @@ func GetCacheDir(fs afero.Fs, cacheDir string) (string, error) { userCacheDir, err := os.UserCacheDir() if err == nil { cacheDir := filepath.Join(userCacheDir, hugoCacheBase) - if err := fs.Mkdir(cacheDir, 0777); err == nil || os.IsExist(err) { + if err := fs.Mkdir(cacheDir, 0o777); err == nil || os.IsExist(err) { return cacheDir, nil } } @@ -494,12 +426,3 @@ func IsEmpty(path string, fs afero.Fs) (bool, error) { func Exists(path string, fs afero.Fs) (bool, error) { return afero.Exists(fs, path) } - -// AddTrailingSlash adds a trailing Unix styled slash (/) if not already -// there. -func AddTrailingSlash(path string) string { - if !strings.HasSuffix(path, "/") { - path += "/" - } - return path -} diff --git a/helpers/path_test.go b/helpers/path_test.go index 45b692923..6f3699589 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -134,7 +134,6 @@ func TestMakePathRelative(t *testing.T) { } func TestGetDottedRelativePath(t *testing.T) { - // on Windows this will receive both kinds, both country and western ... for _, f := range []func(string) string{filepath.FromSlash, func(s string) string { return s }} { doTestGetDottedRelativePath(f, t) } @@ -258,7 +257,7 @@ func createNonZeroSizedFileInTempDir(t *testing.T) *os.File { f := createZeroSizedFileInTempDir(t) byteString := []byte("byteString") - err := os.WriteFile(f.Name(), byteString, 0644) + err := os.WriteFile(f.Name(), byteString, 0o644) if err != nil { t.Error(err) } diff --git a/helpers/pathspec.go b/helpers/pathspec.go index c9bb49038..88571b93c 100644 --- a/helpers/pathspec.go +++ b/helpers/pathspec.go @@ -74,9 +74,5 @@ func NewPathSpecWithBaseBaseFsProvided(fs *hugofs.Fs, cfg config.AllProvider, lo // PermalinkForBaseURL creates a permalink from the given link and baseURL. func (p *PathSpec) PermalinkForBaseURL(link, baseURL string) string { - link = strings.TrimPrefix(link, "/") - if !strings.HasSuffix(baseURL, "/") { - baseURL += "/" - } - return baseURL + link + return baseURL + strings.TrimPrefix(link, "/") } diff --git a/helpers/processing_stats.go b/helpers/processing_stats.go index 3e3e9a3ca..540060aa2 100644 --- a/helpers/processing_stats.go +++ b/helpers/processing_stats.go @@ -31,7 +31,6 @@ type ProcessingStats struct { ProcessedImages uint64 Files uint64 Aliases uint64 - Sitemaps uint64 Cleaned uint64 } @@ -48,7 +47,6 @@ func (s *ProcessingStats) toVals() []processingStatsTitleVal { {"Static files", s.Static}, {"Processed images", s.ProcessedImages}, {"Aliases", s.Aliases}, - {"Sitemaps", s.Sitemaps}, {"Cleaned", s.Cleaned}, } } diff --git a/helpers/url.go b/helpers/url.go index 7d86c529c..d5a613029 100644 --- a/helpers/url.go +++ b/helpers/url.go @@ -20,55 +20,8 @@ import ( "strings" "github.com/gohugoio/hugo/common/paths" - - "github.com/PuerkitoBio/purell" ) -func sanitizeURLWithFlags(in string, f purell.NormalizationFlags) string { - s, err := purell.NormalizeURLString(in, f) - if err != nil { - return in - } - - // Temporary workaround for the bug fix and resulting - // behavioral change in purell.NormalizeURLString(): - // a leading '/' was inadvertently added to relative links, - // but no longer, see #878. - // - // I think the real solution is to allow Hugo to - // make relative URL with relative path, - // e.g. "../../post/hello-again/", as wished by users - // in issues #157, #622, etc., without forcing - // relative URLs to begin with '/'. - // Once the fixes are in, let's remove this kludge - // and restore SanitizeURL() to the way it was. - // -- @anthonyfok, 2015-02-16 - // - // Begin temporary kludge - u, err := url.Parse(s) - if err != nil { - panic(err) - } - if len(u.Path) > 0 && !strings.HasPrefix(u.Path, "/") { - u.Path = "/" + u.Path - } - return u.String() - // End temporary kludge - - // return s - -} - -// SanitizeURL sanitizes the input URL string. -func SanitizeURL(in string) string { - return sanitizeURLWithFlags(in, purell.FlagsSafe|purell.FlagRemoveTrailingSlash|purell.FlagRemoveDotSegments|purell.FlagRemoveDuplicateSlashes|purell.FlagRemoveUnnecessaryHostDots|purell.FlagRemoveEmptyPortSeparator) -} - -// SanitizeURLKeepTrailingSlash is the same as SanitizeURL, but will keep any trailing slash. -func SanitizeURLKeepTrailingSlash(in string) string { - return sanitizeURLWithFlags(in, purell.FlagsSafe|purell.FlagRemoveDotSegments|purell.FlagRemoveDuplicateSlashes|purell.FlagRemoveUnnecessaryHostDots|purell.FlagRemoveEmptyPortSeparator) -} - // URLize is similar to MakePath, but with Unicode handling // Example: // diff --git a/helpers/url_test.go b/helpers/url_test.go index 448756b5b..ce1b24487 100644 --- a/helpers/url_test.go +++ b/helpers/url_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,7 +20,6 @@ import ( qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" ) func TestURLize(t *testing.T) { @@ -193,7 +192,6 @@ func doTestRelURL(t testing.TB, defaultInSubDir, addLanguage, multilingual bool, canonify bool expected string }{ - // Issue 9994 {"/foo/bar", "https://example.org/foo/", false, "MULTI/foo/bar"}, {"foo/bar", "https://example.org/foo/", false, "/fooMULTI/foo/bar"}, @@ -211,7 +209,7 @@ func doTestRelURL(t testing.TB, defaultInSubDir, addLanguage, multilingual bool, {"test/", "http://base/sub/", false, "/subMULTI/test/"}, {"/test/", "http://base/sub/", true, "MULTI/test/"}, {"", "http://base/ace/", false, "/aceMULTI/"}, - {"", "http://base/ace", false, "/aceMULTI"}, + {"", "http://base/ace", false, "/aceMULTI/"}, {"http://abs", "http://base/", false, "http://abs"}, {"//schemaless", "http://base/", false, "//schemaless"}, } @@ -231,7 +229,6 @@ func doTestRelURL(t testing.TB, defaultInSubDir, addLanguage, multilingual bool, for i, test := range tests { c.Run(fmt.Sprintf("%v/defaultInSubDir=%t;addLanguage=%t;multilingual=%t/%s", test, defaultInSubDir, addLanguage, multilingual, lang), func(c *qt.C) { - v.Set("baseURL", test.baseURL) v.Set("canonifyURLs", test.canonify) defaultContentLanguage := lang @@ -255,36 +252,6 @@ func doTestRelURL(t testing.TB, defaultInSubDir, addLanguage, multilingual bool, c.Assert(output, qt.Equals, expected, qt.Commentf("[%d] %s", i, test.input)) }) - - } -} - -func TestSanitizeURL(t *testing.T) { - tests := []struct { - input string - expected string - }{ - {"http://foo.bar/", "http://foo.bar"}, - {"http://foo.bar", "http://foo.bar"}, // issue #1105 - {"http://foo.bar/zoo/", "http://foo.bar/zoo"}, // issue #931 - } - - for i, test := range tests { - o1 := helpers.SanitizeURL(test.input) - o2 := helpers.SanitizeURLKeepTrailingSlash(test.input) - - expected2 := test.expected - - if strings.HasSuffix(test.input, "/") && !strings.HasSuffix(expected2, "/") { - expected2 += "/" - } - - if o1 != test.expected { - t.Errorf("[%d] 1: Expected %#v, got %#v\n", i, test.expected, o1) - } - if o2 != expected2 { - t.Errorf("[%d] 2: Expected %#v, got %#v\n", i, expected2, o2) - } } } |