diff options
26 files changed, 378 insertions, 49 deletions
diff --git a/media/mediaType.go b/media/mediaType.go index eec7a27a8..0bdeb6db7 100644 --- a/media/mediaType.go +++ b/media/mediaType.go @@ -17,6 +17,7 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "sort" "strings" @@ -60,6 +61,42 @@ type SuffixInfo struct { FullSuffix string `json:"fullSuffix"` } +// FromContent resolve the Type primarily using http.DetectContentType. +// If http.DetectContentType resolves to application/octet-stream, a zero Type is returned. +// If http.DetectContentType resolves to text/plain or application/xml, we try to get more specific using types and ext. +func FromContent(types Types, ext string, content []byte) Type { + ext = strings.TrimPrefix(ext, ".") + t := strings.Split(http.DetectContentType(content), ";")[0] + var m Type + if t == "application/octet-stream" { + return m + } + + var found bool + m, found = types.GetByType(t) + if !found { + if t == "text/xml" { + // This is how it's configured in Hugo by default. + m, found = types.GetByType("application/xml") + } + } + + if !found || ext == "" { + return m + } + + if m.Type() == "text/plain" || m.Type() == "application/xml" { + // http.DetectContentType isn't brilliant when it comes to common text formats, so we need to do better. + // For now we say that if it's detected to be a text format and the extension/content type in header reports + // it to be a text format, then we use that. + mm, _, found := types.GetFirstBySuffix(ext) + if found && mm.IsText() { + return mm + } + } + return m +} + // FromStringAndExt creates a Type from a MIME string and a given extension. func FromStringAndExt(t, ext string) (Type, error) { tp, err := fromString(t) @@ -122,6 +159,21 @@ func (m Type) Suffixes() []string { return strings.Split(m.suffixesCSV, ",") } +// IsText returns whether this Type is a text format. +// Note that this may currently return false negatives. +// TODO(bep) improve +func (m Type) IsText() bool { + if m.MainType == "text" { + return true + } + switch m.SubType { + case "javascript", "json", "rss", "xml", "svg", TOMLType.SubType, YAMLType.SubType: + return true + + } + return false +} + func (m *Type) init() { m.FirstSuffix.FullSuffix = "" m.FirstSuffix.Suffix = "" @@ -183,6 +235,10 @@ var ( BMPType = newMediaType("image", "bmp", []string{"bmp"}) WEBPType = newMediaType("image", "webp", []string{"webp"}) + // Common font types + TrueTypeFontType = newMediaType("font", "ttf", []string{"ttf"}) + OpenTypeFontType = newMediaType("font", "otf", []string{"otf"}) + // Common video types AVIType = newMediaType("video", "x-msvideo", []string{"avi"}) MPEGType = newMediaType("video", "mpeg", []string{"mpg", "mpeg"}) @@ -224,6 +280,8 @@ var DefaultTypes = Types{ OGGType, WEBMType, GPPType, + OpenTypeFontType, + TrueTypeFontType, } func init() { diff --git a/media/mediaType_test.go b/media/mediaType_test.go index b33ca174c..f3a06e8ed 100644 --- a/media/mediaType_test.go +++ b/media/mediaType_test.go @@ -15,10 +15,14 @@ package media import ( "encoding/json" + "io/ioutil" + "path/filepath" "sort" + "strings" "testing" qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/paths" ) func TestDefaultTypes(t *testing.T) { @@ -47,6 +51,8 @@ func TestDefaultTypes(t *testing.T) { {XMLType, "application", "xml", "xml", "application/xml", "application/xml"}, {TOMLType, "application", "toml", "toml", "application/toml", "application/toml"}, {YAMLType, "application", "yaml", "yaml", "application/yaml", "application/yaml"}, + {TrueTypeFontType, "font", "ttf", "ttf", "font/ttf", "font/ttf"}, + {OpenTypeFontType, "font", "otf", "otf", "font/otf", "font/otf"}, } { c.Assert(test.tp.MainType, qt.Equals, test.expectedMainType) c.Assert(test.tp.SubType, qt.Equals, test.expectedSubType) @@ -56,7 +62,7 @@ func TestDefaultTypes(t *testing.T) { } - c.Assert(len(DefaultTypes), qt.Equals, 28) + c.Assert(len(DefaultTypes), qt.Equals, 30) } func TestGetByType(t *testing.T) { @@ -175,6 +181,26 @@ func TestFromExtensionMultipleSuffixes(t *testing.T) { } +func TestFromContent(t *testing.T) { + c := qt.New(t) + + files, err := filepath.Glob("./testdata/resource.*") + c.Assert(err, qt.IsNil) + mtypes := DefaultTypes + + for _, filename := range files { + c.Run(filepath.Base(filename), func(c *qt.C) { + content, err := ioutil.ReadFile(filename) + c.Assert(err, qt.IsNil) + ext := strings.TrimPrefix(paths.Ext(filename), ".") + expected, _, found := mtypes.GetFirstBySuffix(ext) + c.Assert(found, qt.IsTrue) + got := FromContent(mtypes, ext, content) + c.Assert(got, qt.Equals, expected) + }) + } +} + func TestDecodeTypes(t *testing.T) { c := qt.New(t) diff --git a/media/testdata/reosurce.otf b/media/testdata/reosurce.otf Binary files differnew file mode 100644 index 000000000..99034a2de --- /dev/null +++ b/media/testdata/reosurce.otf diff --git a/media/testdata/resource.css b/media/testdata/resource.css new file mode 100644 index 000000000..a267873b5 --- /dev/null +++ b/media/testdata/resource.css @@ -0,0 +1,8 @@ +body { + background-color: lightblue; + } + + h1 { + color: navy; + margin-left: 20px; + }
\ No newline at end of file diff --git a/media/testdata/resource.csv b/media/testdata/resource.csv new file mode 100644 index 000000000..ee6b058b6 --- /dev/null +++ b/media/testdata/resource.csv @@ -0,0 +1,130 @@ +"LatD", "LatM", "LatS", "NS", "LonD", "LonM", "LonS", "EW", "City", "State" + 41, 5, 59, "N", 80, 39, 0, "W", "Youngstown", OH + 42, 52, 48, "N", 97, 23, 23, "W", "Yankton", SD + 46, 35, 59, "N", 120, 30, 36, "W", "Yakima", WA + 42, 16, 12, "N", 71, 48, 0, "W", "Worcester", MA + 43, 37, 48, "N", 89, 46, 11, "W", "Wisconsin Dells", WI + 36, 5, 59, "N", 80, 15, 0, "W", "Winston-Salem", NC + 49, 52, 48, "N", 97, 9, 0, "W", "Winnipeg", MB + 39, 11, 23, "N", 78, 9, 36, "W", "Winchester", VA + 34, 14, 24, "N", 77, 55, 11, "W", "Wilmington", NC + 39, 45, 0, "N", 75, 33, 0, "W", "Wilmington", DE + 48, 9, 0, "N", 103, 37, 12, "W", "Williston", ND + 41, 15, 0, "N", 77, 0, 0, "W", "Williamsport", PA + 37, 40, 48, "N", 82, 16, 47, "W", "Williamson", WV + 33, 54, 0, "N", 98, 29, 23, "W", "Wichita Falls", TX + 37, 41, 23, "N", 97, 20, 23, "W", "Wichita", KS + 40, 4, 11, "N", 80, 43, 12, "W", "Wheeling", WV + 26, 43, 11, "N", 80, 3, 0, "W", "West Palm Beach", FL + 47, 25, 11, "N", 120, 19, 11, "W", "Wenatchee", WA + 41, 25, 11, "N", 122, 23, 23, "W", "Weed", CA + 31, 13, 11, "N", 82, 20, 59, "W", "Waycross", GA + 44, 57, 35, "N", 89, 38, 23, "W", "Wausau", WI + 42, 21, 36, "N", 87, 49, 48, "W", "Waukegan", IL + 44, 54, 0, "N", 97, 6, 36, "W", "Watertown", SD + 43, 58, 47, "N", 75, 55, 11, "W", "Watertown", NY + 42, 30, 0, "N", 92, 20, 23, "W", "Waterloo", IA + 41, 32, 59, "N", 73, 3, 0, "W", "Waterbury", CT + 38, 53, 23, "N", 77, 1, 47, "W", "Washington", DC + 41, 50, 59, "N", 79, 8, 23, "W", "Warren", PA + 46, 4, 11, "N", 118, 19, 48, "W", "Walla Walla", WA + 31, 32, 59, "N", 97, 8, 23, "W", "Waco", TX + 38, 40, 48, "N", 87, 31, 47, "W", "Vincennes", IN + 28, 48, 35, "N", 97, 0, 36, "W", "Victoria", TX + 32, 20, 59, "N", 90, 52, 47, "W", "Vicksburg", MS + 49, 16, 12, "N", 123, 7, 12, "W", "Vancouver", BC + 46, 55, 11, "N", 98, 0, 36, "W", "Valley City", ND + 30, 49, 47, "N", 83, 16, 47, "W", "Valdosta", GA + 43, 6, 36, "N", 75, 13, 48, "W", "Utica", NY + 39, 54, 0, "N", 79, 43, 48, "W", "Uniontown", PA + 32, 20, 59, "N", 95, 18, 0, "W", "Tyler", TX + 42, 33, 36, "N", 114, 28, 12, "W", "Twin Falls", ID + 33, 12, 35, "N", 87, 34, 11, "W", "Tuscaloosa", AL + 34, 15, 35, "N", 88, 42, 35, "W", "Tupelo", MS + 36, 9, 35, "N", 95, 54, 36, "W", "Tulsa", OK + 32, 13, 12, "N", 110, 58, 12, "W", "Tucson", AZ + 37, 10, 11, "N", 104, 30, 36, "W", "Trinidad", CO + 40, 13, 47, "N", 74, 46, 11, "W", "Trenton", NJ + 44, 45, 35, "N", 85, 37, 47, "W", "Traverse City", MI + 43, 39, 0, "N", 79, 22, 47, "W", "Toronto", ON + 39, 2, 59, "N", 95, 40, 11, "W", "Topeka", KS + 41, 39, 0, "N", 83, 32, 24, "W", "Toledo", OH + 33, 25, 48, "N", 94, 3, 0, "W", "Texarkana", TX + 39, 28, 12, "N", 87, 24, 36, "W", "Terre Haute", IN + 27, 57, 0, "N", 82, 26, 59, "W", "Tampa", FL + 30, 27, 0, "N", 84, 16, 47, "W", "Tallahassee", FL + 47, 14, 24, "N", 122, 25, 48, "W", "Tacoma", WA + 43, 2, 59, "N", 76, 9, 0, "W", "Syracuse", NY + 32, 35, 59, "N", 82, 20, 23, "W", "Swainsboro", GA + 33, 55, 11, "N", 80, 20, 59, "W", "Sumter", SC + 40, 59, 24, "N", 75, 11, 24, "W", "Stroudsburg", PA + 37, 57, 35, "N", 121, 17, 24, "W", "Stockton", CA + 44, 31, 12, "N", 89, 34, 11, "W", "Stevens Point", WI + 40, 21, 36, "N", 80, 37, 12, "W", "Steubenville", OH + 40, 37, 11, "N", 103, 13, 12, "W", "Sterling", CO + 38, 9, 0, "N", 79, 4, 11, "W", "Staunton", VA + 39, 55, 11, "N", 83, 48, 35, "W", "Springfield", OH + 37, 13, 12, "N", 93, 17, 24, "W", "Springfield", MO + 42, 5, 59, "N", 72, 35, 23, "W", "Springfield", MA + 39, 47, 59, "N", 89, 39, 0, "W", "Springfield", IL + 47, 40, 11, "N", 117, 24, 36, "W", "Spokane", WA + 41, 40, 48, "N", 86, 15, 0, "W", "South Bend", IN + 43, 32, 24, "N", 96, 43, 48, "W", "Sioux Falls", SD + 42, 29, 24, "N", 96, 23, 23, "W", "Sioux City", IA + 32, 30, 35, "N", 93, 45, 0, "W", "Shreveport", LA + 33, 38, 23, "N", 96, 36, 36, "W", "Sherman", TX + 44, 47, 59, "N", 106, 57, 35, "W", "Sheridan", WY + 35, 13, 47, "N", 96, 40, 48, "W", "Seminole", OK + 32, 25, 11, "N", 87, 1, 11, "W", "Selma", AL + 38, 42, 35, "N", 93, 13, 48, "W", "Sedalia", MO + 47, 35, 59, "N", 122, 19, 48, "W", "Seattle", WA + 41, 24, 35, "N", 75, 40, 11, "W", "Scranton", PA + 41, 52, 11, "N", 103, 39, 36, "W", "Scottsbluff", NB + 42, 49, 11, "N", 73, 56, 59, "W", "Schenectady", NY + 32, 4, 48, "N", 81, 5, 23, "W", "Savannah", GA + 46, 29, 24, "N", 84, 20, 59, "W", "Sault Sainte Marie", MI + 27, 20, 24, "N", 82, 31, 47, "W", "Sarasota", FL + 38, 26, 23, "N", 122, 43, 12, "W", "Santa Rosa", CA + 35, 40, 48, "N", 105, 56, 59, "W", "Santa Fe", NM + 34, 25, 11, "N", 119, 41, 59, "W", "Santa Barbara", CA + 33, 45, 35, "N", 117, 52, 12, "W", "Santa Ana", CA + 37, 20, 24, "N", 121, 52, 47, "W", "San Jose", CA + 37, 46, 47, "N", 122, 25, 11, "W", "San Francisco", CA + 41, 27, 0, "N", 82, 42, 35, "W", "Sandusky", OH + 32, 42, 35, "N", 117, 9, 0, "W", "San Diego", CA + 34, 6, 36, "N", 117, 18, 35, "W", "San Bernardino", CA + 29, 25, 12, "N", 98, 30, 0, "W", "San Antonio", TX + 31, 27, 35, "N", 100, 26, 24, "W", "San Angelo", TX + 40, 45, 35, "N", 111, 52, 47, "W", "Salt Lake City", UT + 38, 22, 11, "N", 75, 35, 59, "W", "Salisbury", MD + 36, 40, 11, "N", 121, 39, 0, "W", "Salinas", CA + 38, 50, 24, "N", 97, 36, 36, "W", "Salina", KS + 38, 31, 47, "N", 106, 0, 0, "W", "Salida", CO + 44, 56, 23, "N", 123, 1, 47, "W", "Salem", OR + 44, 57, 0, "N", 93, 5, 59, "W", "Saint Paul", MN + 38, 37, 11, "N", 90, 11, 24, "W", "Saint Louis", MO + 39, 46, 12, "N", 94, 50, 23, "W", "Saint Joseph", MO + 42, 5, 59, "N", 86, 28, 48, "W", "Saint Joseph", MI + 44, 25, 11, "N", 72, 1, 11, "W", "Saint Johnsbury", VT + 45, 34, 11, "N", 94, 10, 11, "W", "Saint Cloud", MN + 29, 53, 23, "N", 81, 19, 11, "W", "Saint Augustine", FL + 43, 25, 48, "N", 83, 56, 24, "W", "Saginaw", MI + 38, 35, 24, "N", 121, 29, 23, "W", "Sacramento", CA + 43, 36, 36, "N", 72, 58, 12, "W", "Rutland", VT + 33, 24, 0, "N", 104, 31, 47, "W", "Roswell", NM + 35, 56, 23, "N", 77, 48, 0, "W", "Rocky Mount", NC + 41, 35, 24, "N", 109, 13, 48, "W", "Rock Springs", WY + 42, 16, 12, "N", 89, 5, 59, "W", "Rockford", IL + 43, 9, 35, "N", 77, 36, 36, "W", "Rochester", NY + 44, 1, 12, "N", 92, 27, 35, "W", "Rochester", MN + 37, 16, 12, "N", 79, 56, 24, "W", "Roanoke", VA + 37, 32, 24, "N", 77, 26, 59, "W", "Richmond", VA + 39, 49, 48, "N", 84, 53, 23, "W", "Richmond", IN + 38, 46, 12, "N", 112, 5, 23, "W", "Richfield", UT + 45, 38, 23, "N", 89, 25, 11, "W", "Rhinelander", WI + 39, 31, 12, "N", 119, 48, 35, "W", "Reno", NV + 50, 25, 11, "N", 104, 39, 0, "W", "Regina", SA + 40, 10, 48, "N", 122, 14, 23, "W", "Red Bluff", CA + 40, 19, 48, "N", 75, 55, 48, "W", "Reading", PA + 41, 9, 35, "N", 81, 14, 23, "W", "Ravenna", OH + diff --git a/media/testdata/resource.ics b/media/testdata/resource.ics new file mode 100644 index 000000000..b9a263e93 --- /dev/null +++ b/media/testdata/resource.ics @@ -0,0 +1,24 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//ZContent.net//Zap Calendar 1.0//EN +CALSCALE:GREGORIAN +METHOD:PUBLISH +BEGIN:VEVENT +SUMMARY:Abraham Lincoln +UID:c7614cff-3549-4a00-9152-d25cc1fe077d +SEQUENCE:0 +STATUS:CONFIRMED +TRANSP:TRANSPARENT +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=2;BYMONTHDAY=12 +DTSTART:20080212 +DTEND:20080213 +DTSTAMP:20150421T141403 +CATEGORIES:U.S. Presidents,Civil War People +LOCATION:Hodgenville\, Kentucky +GEO:37.5739497;-85.7399606 +DESCRIPTION:Born February 12\, 1809\nSixteenth President (1861-1865)\n\n\n + \nhttp://AmericanHistoryCalendar.com +URL:http://americanhistorycalendar.com/peoplecalendar/1,328-abraham-lincol + n +END:VEVENT +END:VCALENDAR
\ No newline at end of file diff --git a/media/testdata/resource.jpg b/media/testdata/resource.jpg Binary files differnew file mode 100644 index 000000000..a9049e81b --- /dev/null +++ b/media/testdata/resource.jpg diff --git a/media/testdata/resource.js b/media/testdata/resource.js new file mode 100644 index 000000000..75ba3b7fe --- /dev/null +++ b/media/testdata/resource.js @@ -0,0 +1,3 @@ +function foo() { + return "foo"; +}
\ No newline at end of file diff --git a/media/testdata/resource.json b/media/testdata/resource.json new file mode 100644 index 000000000..446899897 --- /dev/null +++ b/media/testdata/resource.json @@ -0,0 +1,14 @@ +{ + "firstName": "Joe", + "lastName": "Jackson", + "gender": "male", + "age": 28, + "address": { + "streetAddress": "101", + "city": "San Diego", + "state": "CA" + }, + "phoneNumbers": [ + { "type": "home", "number": "7349282382" } + ] +}
\ No newline at end of file diff --git a/media/testdata/resource.png b/media/testdata/resource.png Binary files differnew file mode 100644 index 000000000..08ae570d2 --- /dev/null +++ b/media/testdata/resource.png diff --git a/media/testdata/resource.rss b/media/testdata/resource.rss new file mode 100644 index 000000000..b20b0fcca --- /dev/null +++ b/media/testdata/resource.rss @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<rss version="2.0"> + +<channel> + <title>W3Schools Home Page</title> + <link>https://www.w3schools.com</link> + <description>Free web building tutorials</description> + <item> + <title>RSS Tutorial</title> + <link>https://www.w3schools.com/xml/xml_rss.asp</link> + <description>New RSS tutorial on W3Schools</description> + </item> + <item> + <title>XML Tutorial</title> + <link>https://www.w3schools.com/xml</link> + <description>New XML tutorial on W3Schools</description> + </item> +</channel> + +</rss>
\ No newline at end of file diff --git a/media/testdata/resource.sass b/media/testdata/resource.sass new file mode 100644 index 000000000..ad857fac7 --- /dev/null +++ b/media/testdata/resource.sass @@ -0,0 +1,6 @@ +$font-stack: Helvetica, sans-serif +$primary-color: #333 + +body + font: 100% $font-stack + color: $primary-color
\ No newline at end of file diff --git a/media/testdata/resource.scss b/media/testdata/resource.scss new file mode 100644 index 000000000..d63e420f6 --- /dev/null +++ b/media/testdata/resource.scss @@ -0,0 +1,7 @@ +$font-stack: Helvetica, sans-serif; +$primary-color: #333; + +body { + font: 100% $font-stack; + color: $primary-color; +}
\ No newline at end of file diff --git a/media/testdata/resource.svg b/media/testdata/resource.svg new file mode 100644 index 000000000..2759ae703 --- /dev/null +++ b/media/testdata/resource.svg @@ -0,0 +1,5 @@ +<svg height="100" width="100"> + <circle cx="50" cy="50" r="40" stroke="black" stroke-width="3" fill="red" /> + Sorry, your browser does not support inline SVG. +</svg> +
\ No newline at end of file diff --git a/media/testdata/resource.ttf b/media/testdata/resource.ttf Binary files differnew file mode 100644 index 000000000..8bc614d06 --- /dev/null +++ b/media/testdata/resource.ttf diff --git a/media/testdata/resource.webp b/media/testdata/resource.webp Binary files differnew file mode 100644 index 000000000..4365e7b9f --- /dev/null +++ b/media/testdata/resource.webp diff --git a/media/testdata/resource.xml b/media/testdata/resource.xml new file mode 100644 index 000000000..fa0c0a5b6 --- /dev/null +++ b/media/testdata/resource.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="UTF-8"?> +<note> + <to>Tove</to> + <from>Jani</from> + <heading>Reminder</heading> + <body>Don't forget me this weekend!</body> +</note>
\ No newline at end of file diff --git a/resources/images/config.go b/resources/images/config.go index c8990d5ca..a8b5412d6 100644 --- a/resources/images/config.go +++ b/resources/images/config.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/media" "github.com/pkg/errors" @@ -45,6 +46,15 @@ var ( ".webp": WEBP, } + imageFormatsBySubType = map[string]Format{ + media.JPEGType.SubType: JPEG, + media.PNGType.SubType: PNG, + media.TIFFType.SubType: TIFF, + media.BMPType.SubType: BMP, + media.GIFType.SubType: GIF, + media.WEBPType.SubType: WEBP, + } + // Add or increment if changes to an image format's processing requires // re-generation. imageFormatsVersions = map[Format]int{ @@ -102,6 +112,11 @@ func ImageFormatFromExt(ext string) (Format, bool) { return f, found } +func ImageFormatFromMediaSubType(sub string) (Format, bool) { + f, found := imageFormatsBySubType[sub] + return f, found +} + const ( defaultJPEGQuality = 75 defaultResampleFilter = "box" diff --git a/resources/images/filters.go b/resources/images/filters.go index e166a0f9d..fd7e31457 100644 --- a/resources/images/filters.go +++ b/resources/images/filters.go @@ -66,6 +66,9 @@ func (*Filters) Text(text string, options ...interface{}) gift.Filter { case "linespacing": tf.linespacing = cast.ToInt(v) case "font": + if err, ok := v.(error); ok { + panic(fmt.Sprintf("invalid font source: %s", err)) + } fontSource, ok1 := v.(hugio.ReadSeekCloserProvider) identifier, ok2 := v.(resource.Identifier) diff --git a/resources/postpub/fields_test.go b/resources/postpub/fields_test.go index 19c3720f7..c408e7791 100644 --- a/resources/postpub/fields_test.go +++ b/resources/postpub/fields_test.go @@ -36,6 +36,7 @@ func TestCreatePlaceholders(t *testing.T) { "Suffixes": "pre_foo.Suffixes_post", "Delimiter": "pre_foo.Delimiter_post", "FirstSuffix": "pre_foo.FirstSuffix_post", + "IsText": "pre_foo.IsText_post", "String": "pre_foo.String_post", "Type": "pre_foo.Type_post", "MainType": "pre_foo.MainType_post", diff --git a/resources/resource.go b/resources/resource.go index 1f6246859..4bf35f9ac 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -69,6 +69,9 @@ type ResourceSourceDescriptor struct { Fs afero.Fs + // Set when its known up front, else it's resolved from the target filename. + MediaType media.Type + // The relative target filename without any language code. RelTargetFilename string diff --git a/resources/resource_factories/create/remote.go b/resources/resource_factories/create/remote.go index 53e77bc5e..f6d3f13dd 100644 --- a/resources/resource_factories/create/remote.go +++ b/resources/resource_factories/create/remote.go @@ -29,6 +29,7 @@ import ( "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" "github.com/mitchellh/mapstructure" @@ -99,7 +100,7 @@ func (c *Client) FromRemote(uri string, optionsm map[string]interface{}) (resour body, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, errors.Wrapf(err, "failed to read remote resource %s", uri) + return nil, errors.Wrapf(err, "failed to read remote resource %q", uri) } filename := path.Base(rURL.Path) @@ -109,33 +110,30 @@ func (c *Client) FromRemote(uri string, optionsm map[string]interface{}) (resour } } - var extension string + var extensionHint string + if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 { - extension = arr[0] + extensionHint = arr[0] } - // If extension was not determined by header, look for a file extention - if extension == "" { + // Look for a file extention + if extensionHint == "" { if ext := path.Ext(filename); ext != "" { - extension = ext + extensionHint = ext } } - // If extension was not determined by header or file extention, try using content itself - if extension == "" { - if ct := http.DetectContentType(body); ct != "application/octet-stream" { - if ct == "image/jpeg" { - extension = ".jpg" - } else if arr, _ := mime.ExtensionsByType(ct); arr != nil { - extension = arr[0] - } - } + // Now resolve the media type primarily using the content. + mediaType := media.FromContent(c.rs.MediaTypes, extensionHint, body) + if mediaType.IsZero() { + return nil, errors.Errorf("failed to resolve media type for remote resource %q", uri) } - resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + extension + resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + mediaType.FirstSuffix.FullSuffix return c.rs.New( resources.ResourceSourceDescriptor{ + MediaType: mediaType, LazyPublish: true, OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil diff --git a/resources/resource_spec.go b/resources/resource_spec.go index 897c1bbaa..cd1e5010d 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -272,21 +272,28 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso fd.RelTargetFilename = sourceFilename } - ext := strings.ToLower(filepath.Ext(fd.RelTargetFilename)) - mimeType, suffixInfo, found := r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, ".")) - // TODO(bep) we need to handle these ambiguous types better, but in this context - // we most likely want the application/xml type. - if suffixInfo.Suffix == "xml" && mimeType.SubType == "rss" { - mimeType, found = r.MediaTypes.GetByType("application/xml") - } + mimeType := fd.MediaType + if mimeType.IsZero() { + ext := strings.ToLower(filepath.Ext(fd.RelTargetFilename)) + var ( + found bool + suffixInfo media.SuffixInfo + ) + mimeType, suffixInfo, found = r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, ".")) + // TODO(bep) we need to handle these ambiguous types better, but in this context + // we most likely want the application/xml type. + if suffixInfo.Suffix == "xml" && mimeType.SubType == "rss" { + mimeType, found = r.MediaTypes.GetByType("application/xml") + } - if !found { - // A fallback. Note that mime.TypeByExtension is slow by Hugo standards, - // so we should configure media types to avoid this lookup for most - // situations. - mimeStr := mime.TypeByExtension(ext) - if mimeStr != "" { - mimeType, _ = media.FromStringAndExt(mimeStr, ext) + if !found { + // A fallback. Note that mime.TypeByExtension is slow by Hugo standards, + // so we should configure media types to avoid this lookup for most + // situations. + mimeStr := mime.TypeByExtension(ext) + if mimeStr != "" { + mimeType, _ = media.FromStringAndExt(mimeStr, ext) + } } } @@ -301,7 +308,7 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso mimeType) if mimeType.MainType == "image" { - imgFormat, ok := images.ImageFormatFromExt(ext) + imgFormat, ok := images.ImageFormatFromMediaSubType(mimeType.SubType) if ok { ir := &imageResource{ Image: images.NewImage(imgFormat, r.imaging, nil, gr), diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go index 4433e56e5..8cd670603 100644 --- a/tpl/resources/resources.go +++ b/tpl/resources/resources.go @@ -110,30 +110,21 @@ func (ns *Namespace) getscssClientDartSass() (*dartsass.Client, error) { // Get locates the filename given in Hugo's assets filesystem and // creates a Resource object that can be used for // further transformations. -func (ns *Namespace) Get(filename interface{}) resource.Resource { - get := func(args ...interface{}) (resource.Resource, error) { - filenamestr, err := cast.ToStringE(filename) - if err != nil { - return nil, err - } - return ns.createClient.Get(filepath.Clean(filenamestr)) - } - - r, err := get(filename) +func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) { + filenamestr, err := cast.ToStringE(filename) if err != nil { - // This allows the client to reason about the .Err in the template. - // This is not as relevant for local resources as remotes, but - // it makes this method work the same way as resources.GetRemote. - return resources.NewErrorResource(errors.Wrap(err, "error calling resources.Get")) + return nil, err } - return r - + return ns.createClient.Get(filepath.Clean(filenamestr)) } // GetRemote gets the URL (via HTTP(s)) in the first argument in args and creates Resource object that can be used for // further transformations. // // A second argument may be provided with an option map. +// +// Note: This method does not return any error as a second argument, +// for any error situations the error can be checked in .Err. func (ns *Namespace) GetRemote(args ...interface{}) resource.Resource { get := func(args ...interface{}) (resource.Resource, error) { if len(args) < 1 { diff --git a/tpl/tplimpl/template_funcs_test.go b/tpl/tplimpl/template_funcs_test.go index 6ddf13b76..711d1350d 100644 --- a/tpl/tplimpl/template_funcs_test.go +++ b/tpl/tplimpl/template_funcs_test.go @@ -37,7 +37,6 @@ import ( "github.com/gohugoio/hugo/tpl/internal" "github.com/gohugoio/hugo/tpl/partials" "github.com/spf13/afero" - ) var logger = loggers.NewErrorLogger() diff --git a/tpl/transform/unmarshal.go b/tpl/transform/unmarshal.go index aa84ca1f8..c59269577 100644 --- a/tpl/transform/unmarshal.go +++ b/tpl/transform/unmarshal.go @@ -95,6 +95,10 @@ func (ns *Namespace) Unmarshal(args ...interface{}) (interface{}, error) { return nil, errors.Errorf("type %T not supported", data) } + if dataStr == "" { + return nil, errors.New("no data to transform") + } + key := helpers.MD5String(dataStr) return ns.cache.GetOrCreate(key, func() (interface{}, error) { |
