media: Allow multiple file suffixes per media type

Before this commit, `Suffix` on `MediaType` was used both to set a custom file suffix and as a way to augment the mediatype definition (what you see after the "+", e.g. "image/svg+xml").

This had its limitations. For one, it was only possible with one file extension per MIME type.

Now you can specify multiple file suffixes using "suffixes", but you need to specify the full MIME type
identifier:

[mediaTypes]
[mediaTypes."image/svg+xml"]
suffixes = ["svg", "abc ]

In most cases, it will be enough to just change:

[mediaTypes]
[mediaTypes."my/custom-mediatype"]
suffix = "txt"

To:

[mediaTypes]
[mediaTypes."my/custom-mediatype"]
suffixes = ["txt"]

Hugo will still respect values set in "suffix" if no value for "suffixes" is provided, but this will be removed in a future release.

Note that you can still get the Media Type's suffix from a template: {{ $mediaType.Suffix }}. But this will now map to the MIME type filename.

Fixes #4920

2 files changed, 278 insertions, 94 deletions

diff --git a/media/mediaType.go b/media/mediaType.go
index 07ba410fb..f26209ea1 100644
--- a/media/mediaType.go
+++ b/media/mediaType.go
@@ -19,6 +19,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/gohugoio/hugo/helpers"
 	"github.com/mitchellh/mapstructure"
 )
 
@@ -29,19 +30,39 @@ const (
 // Type (also known as MIME type and content type) is a two-part identifier for
 // file formats and format contents transmitted on the Internet.
 // For Hugo's use case, we use the top-level type name / subtype name + suffix.
-// One example would be image/jpeg+jpg
+// One example would be application/svg+xml
 // If suffix is not provided, the sub type will be used.
 // See // https://en.wikipedia.org/wiki/Media_type
 type Type struct {
-	MainType  string `json:"mainType"`  // i.e. text
-	SubType   string `json:"subType"`   // i.e. html
-	Suffix    string `json:"suffix"`    // i.e html
-	Delimiter string `json:"delimiter"` // defaults to "."
+	MainType string `json:"mainType"` // i.e. text
+	SubType  string `json:"subType"`  // i.e. html
+
+	// Deprecated in Hugo 0.44. To be renamed and unexported.
+	// Was earlier used both to set file suffix and to augment the MIME type.
+	// This had its limitations and issues.
+	OldSuffix string `json:"-" mapstructure:"suffix"`
+
+	Delimiter string `json:"delimiter"` // e.g. "."
+
+	Suffixes []string `json:"suffixes"`
+
+	// Set when doing lookup by suffix.
+	fileSuffix string
 }
 
-// FromString creates a new Type given a type sring on the form MainType/SubType and
+// FromStringAndExt is same as FromString, but adds the file extension to the type.
+func FromStringAndExt(t, ext string) (Type, error) {
+	tp, err := fromString(t)
+	if err != nil {
+		return tp, err
+	}
+	tp.Suffixes = []string{strings.TrimPrefix(ext, ".")}
+	return tp, nil
+}
+
+// FromString creates a new Type given a type string on the form MainType/SubType and
 // an optional suffix, e.g. "text/html" or "text/html+html".
-func FromString(t string) (Type, error) {
+func fromString(t string) (Type, error) {
 	t = strings.ToLower(t)
 	parts := strings.Split(t, "/")
 	if len(parts) != 2 {
@@ -54,54 +75,67 @@ func FromString(t string) (Type, error) {
 
 	var suffix string
 
-	if len(subParts) == 1 {
-		suffix = subType
-	} else {
+	if len(subParts) > 1 {
 		suffix = subParts[1]
 	}
 
-	return Type{MainType: mainType, SubType: subType, Suffix: suffix, Delimiter: defaultDelimiter}, nil
+	return Type{MainType: mainType, SubType: subType, OldSuffix: suffix}, nil
 }
 
-// Type returns a string representing the main- and sub-type of a media type, i.e. "text/css".
+// Type returns a string representing the main- and sub-type of a media type, e.g. "text/css".
+// A suffix identifier will be appended after a "+" if set, e.g. "image/svg+xml".
 // Hugo will register a set of default media types.
 // These can be overridden by the user in the configuration,
 // by defining a media type with the same Type.
 func (m Type) Type() string {
+	// Examples are
+	// image/svg+xml
+	// text/css
+	if m.OldSuffix != "" {
+		return fmt.Sprintf("%s/%s+%s", m.MainType, m.SubType, m.OldSuffix)
+	}
 	return fmt.Sprintf("%s/%s", m.MainType, m.SubType)
+
 }
 
 func (m Type) String() string {
-	if m.Suffix != "" {
-		return fmt.Sprintf("%s/%s+%s", m.MainType, m.SubType, m.Suffix)
-	}
-	return fmt.Sprintf("%s/%s", m.MainType, m.SubType)
+	return m.Type()
 }
 
 // FullSuffix returns the file suffix with any delimiter prepended.
 func (m Type) FullSuffix() string {
-	return m.Delimiter + m.Suffix
+	return m.Delimiter + m.Suffix()
+}
+
+// Suffix returns the file suffix without any delmiter prepended.
+func (m Type) Suffix() string {
+	if m.fileSuffix != "" {
+		return m.fileSuffix
+	}
+	if len(m.Suffixes) > 0 {
+		return m.Suffixes[0]
+	}
+	// There are MIME types without file suffixes.
+	return ""
 }
 
 var (
-	CalendarType   = Type{"text", "calendar", "ics", defaultDelimiter}
-	CSSType        = Type{"text", "css", "css", defaultDelimiter}
-	SCSSType       = Type{"text", "x-scss", "scss", defaultDelimiter}
-	SASSType       = Type{"text", "x-sass", "sass", defaultDelimiter}
-	CSVType        = Type{"text", "csv", "csv", defaultDelimiter}
-	HTMLType       = Type{"text", "html", "html", defaultDelimiter}
-	JavascriptType = Type{"application", "javascript", "js", defaultDelimiter}
-	JSONType       = Type{"application", "json", "json", defaultDelimiter}
-	RSSType        = Type{"application", "rss", "xml", defaultDelimiter}
-	XMLType        = Type{"application", "xml", "xml", defaultDelimiter}
-	// The official MIME type of SVG is image/svg+xml. We currently only support one extension
-	// per mime type. The workaround in projects is to create multiple media type definitions,
-	// but we need to improve this to take other known suffixes into account.
-	// But until then, svg has an svg extension, which is very common. TODO(bep)
-	SVGType  = Type{"image", "svg", "svg", defaultDelimiter}
-	TextType = Type{"text", "plain", "txt", defaultDelimiter}
-
-	OctetType = Type{"application", "octet-stream", "", ""}
+	// Definitions from https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types etc.
+	// Note that from Hugo 0.44 we only set Suffix if it is part of the MIME type.
+	CalendarType   = Type{MainType: "text", SubType: "calendar", Suffixes: []string{"ics"}, Delimiter: defaultDelimiter}
+	CSSType        = Type{MainType: "text", SubType: "css", Suffixes: []string{"css"}, Delimiter: defaultDelimiter}
+	SCSSType       = Type{MainType: "text", SubType: "x-scss", Suffixes: []string{"scss"}, Delimiter: defaultDelimiter}
+	SASSType       = Type{MainType: "text", SubType: "x-sass", Suffixes: []string{"sass"}, Delimiter: defaultDelimiter}
+	CSVType        = Type{MainType: "text", SubType: "csv", Suffixes: []string{"csv"}, Delimiter: defaultDelimiter}
+	HTMLType       = Type{MainType: "text", SubType: "html", Suffixes: []string{"html"}, Delimiter: defaultDelimiter}
+	JavascriptType = Type{MainType: "application", SubType: "javascript", Suffixes: []string{"js"}, Delimiter: defaultDelimiter}
+	JSONType       = Type{MainType: "application", SubType: "json", Suffixes: []string{"json"}, Delimiter: defaultDelimiter}
+	RSSType        = Type{MainType: "application", SubType: "rss", OldSuffix: "xml", Suffixes: []string{"xml"}, Delimiter: defaultDelimiter}
+	XMLType        = Type{MainType: "application", SubType: "xml", Suffixes: []string{"xml"}, Delimiter: defaultDelimiter}
+	SVGType        = Type{MainType: "image", SubType: "svg", OldSuffix: "xml", Suffixes: []string{"svg"}, Delimiter: defaultDelimiter}
+	TextType       = Type{MainType: "text", SubType: "plain", Suffixes: []string{"txt"}, Delimiter: defaultDelimiter}
+
+	OctetType = Type{MainType: "application", SubType: "octet-stream"}
 )
 
 var DefaultTypes = Types{
@@ -136,13 +170,23 @@ func (t Types) GetByType(tp string) (Type, bool) {
 			return tt, true
 		}
 	}
+
+	if !strings.Contains(tp, "+") {
+		// Try with the main and sub type
+		parts := strings.Split(tp, "/")
+		if len(parts) == 2 {
+			return t.GetByMainSubType(parts[0], parts[1])
+		}
+	}
+
 	return Type{}, false
 }
 
 // GetFirstBySuffix will return the first media type matching the given suffix.
 func (t Types) GetFirstBySuffix(suffix string) (Type, bool) {
 	for _, tt := range t {
-		if strings.EqualFold(suffix, tt.Suffix) {
+		if match := tt.matchSuffix(suffix); match != "" {
+			tt.fileSuffix = match
 			return tt, true
 		}
 	}
@@ -155,59 +199,146 @@ func (t Types) GetFirstBySuffix(suffix string) (Type, bool) {
 // The lookup is case insensitive.
 func (t Types) GetBySuffix(suffix string) (tp Type, found bool) {
 	for _, tt := range t {
-		if strings.EqualFold(suffix, tt.Suffix) {
+		if match := tt.matchSuffix(suffix); match != "" {
 			if found {
 				// ambiguous
 				found = false
 				return
 			}
 			tp = tt
+			tp.fileSuffix = match
 			found = true
 		}
 	}
 	return
 }
 
+func (t Type) matchSuffix(suffix string) string {
+	if strings.EqualFold(suffix, t.OldSuffix) {
+		return t.OldSuffix
+	}
+	for _, s := range t.Suffixes {
+		if strings.EqualFold(suffix, s) {
+			return s
+		}
+	}
+
+	return ""
+}
+
+// GetMainSubType gets a media type given a main and a sub type e.g. "text" and "plain".
+// It will return false if no format could be found, or if the combination given
+// is ambiguous.
+// The lookup is case insensitive.
+func (t Types) GetByMainSubType(mainType, subType string) (tp Type, found bool) {
+	for _, tt := range t {
+		if strings.EqualFold(mainType, tt.MainType) && strings.EqualFold(subType, tt.SubType) {
+			if found {
+				// ambiguous
+				found = false
+				return
+			}
+
+			tp = tt
+			found = true
+		}
+	}
+	return
+}
+
+func suffixIsDeprecated() {
+	helpers.Deprecated("MediaType", "Suffix in config.toml", `
+Before Hugo 0.44 this was used both to set a custom file suffix and as way
+to augment the mediatype definition (what you see after the "+", e.g. "image/svg+xml").
+
+This had its limitations. For one, it was only possible with one file extension per MIME type.
+
+Now you can specify multiple file suffixes using "suffixes", but you need to specify the full MIME type
+identifier:
+
+[mediaTypes]
+[mediaTypes."image/svg+xml"]
+suffixes = ["svg", "abc" ]
+
+In most cases, it will be enough to just change:
+
+[mediaTypes]
+[mediaTypes."my/custom-mediatype"]
+suffix = "txt"
+
+To:
+
+[mediaTypes]
+[mediaTypes."my/custom-mediatype"]
+suffixes = ["txt"]
+
+Hugo will still respect values set in "suffix" if no value for "suffixes" is provided, but this will be removed
+in a future release.
+
+Note that you can still get the Media Type's suffix from a template: {{ $mediaType.Suffix }}. But this will now map to the MIME type filename.
+`, false)
+}
+
 // DecodeTypes takes a list of media type configurations and merges those,
 // in the order given, with the Hugo defaults as the last resort.
 func DecodeTypes(maps ...map[string]interface{}) (Types, error) {
-	m := make(Types, len(DefaultTypes))
-	copy(m, DefaultTypes)
+	var m Types
+
+	// Maps type string to Type. Type string is the full application/svg+xml.
+	mmm := make(map[string]Type)
+	for _, dt := range DefaultTypes {
+		suffixes := make([]string, len(dt.Suffixes))
+		copy(suffixes, dt.Suffixes)
+		dt.Suffixes = suffixes
+		mmm[dt.Type()] = dt
+	}
 
 	for _, mm := range maps {
 		for k, v := range mm {
-			// It may be tempting to put the full media type in the key, e.g.
-			//  "text/css+css", but that will break the logic below.
-			if strings.Contains(k, "+") {
-				return Types{}, fmt.Errorf("media type keys cannot contain any '+' chars. Valid example is %q", "text/css")
-			}
+			var mediaType Type
 
-			found := false
-			for i, vv := range m {
-				// Match by type, i.e. "text/css"
-				if strings.EqualFold(k, vv.Type()) {
-					// Merge it with the existing
-					if err := mapstructure.WeakDecode(v, &m[i]); err != nil {
-						return m, err
-					}
-					found = true
-				}
-			}
+			mediaType, found := mmm[k]
 			if !found {
-				mediaType, err := FromString(k)
+				var err error
+				mediaType, err = fromString(k)
 				if err != nil {
 					return m, err
 				}
+			}
 
-				if err := mapstructure.WeakDecode(v, &mediaType); err != nil {
-					return m, err
-				}
+			if err := mapstructure.WeakDecode(v, &mediaType); err != nil {
+				return m, err
+			}
+
+			vm := v.(map[string]interface{})
+			_, delimiterSet := vm["delimiter"]
+			_, suffixSet := vm["suffix"]
+
+			if mediaType.OldSuffix != "" {
+				suffixIsDeprecated()
+			}
 
-				m = append(m, mediaType)
+			// Before Hugo 0.44 we had a non-standard use of the Suffix
+			// attribute, and this is now deprecated (use Suffixes for file suffixes).
+			// But we need to keep old configurations working for a while.
+			if len(mediaType.Suffixes) == 0 && mediaType.OldSuffix != "" {
+				mediaType.Suffixes = []string{mediaType.OldSuffix}
 			}
+			// The user may set the delimiter as an empty string.
+			if !delimiterSet && len(mediaType.Suffixes) != 0 {
+				mediaType.Delimiter = defaultDelimiter
+			} else if suffixSet && !delimiterSet {
+				mediaType.Delimiter = defaultDelimiter
+			}
+
+			mmm[k] = mediaType
+
 		}
 	}
 
+	for _, v := range mmm {
+		m = append(m, v)
+	}
 	sort.Sort(m)
 
 	return m, nil
diff --git a/media/mediaType_test.go b/media/mediaType_test.go
index f3ddb086c..6385528ee 100644
--- a/media/mediaType_test.go
+++ b/media/mediaType_test.go
@@ -28,21 +28,21 @@ func TestDefaultTypes(t *testing.T) {
 		expectedType     string
 		expectedString   string
 	}{
-		{CalendarType, "text", "calendar", "ics", "text/calendar", "text/calendar+ics"},
-		{CSSType, "text", "css", "css", "text/css", "text/css+css"},
-		{SCSSType, "text", "x-scss", "scss", "text/x-scss", "text/x-scss+scss"},
-		{CSVType, "text", "csv", "csv", "text/csv", "text/csv+csv"},
-		{HTMLType, "text", "html", "html", "text/html", "text/html+html"},
-		{JavascriptType, "application", "javascript", "js", "application/javascript", "application/javascript+js"},
-		{JSONType, "application", "json", "json", "application/json", "application/json+json"},
-		{RSSType, "application", "rss", "xml", "application/rss", "application/rss+xml"},
-		{SVGType, "image", "svg", "svg", "image/svg", "image/svg+svg"},
-		{TextType, "text", "plain", "txt", "text/plain", "text/plain+txt"},
-		{XMLType, "application", "xml", "xml", "application/xml", "application/xml+xml"},
+		{CalendarType, "text", "calendar", "ics", "text/calendar", "text/calendar"},
+		{CSSType, "text", "css", "css", "text/css", "text/css"},
+		{SCSSType, "text", "x-scss", "scss", "text/x-scss", "text/x-scss"},
+		{CSVType, "text", "csv", "csv", "text/csv", "text/csv"},
+		{HTMLType, "text", "html", "html", "text/html", "text/html"},
+		{JavascriptType, "application", "javascript", "js", "application/javascript", "application/javascript"},
+		{JSONType, "application", "json", "json", "application/json", "application/json"},
+		{RSSType, "application", "rss", "xml", "application/rss+xml", "application/rss+xml"},
+		{SVGType, "image", "svg", "svg", "image/svg+xml", "image/svg+xml"},
+		{TextType, "text", "plain", "txt", "text/plain", "text/plain"},
+		{XMLType, "application", "xml", "xml", "application/xml", "application/xml"},
 	} {
 		require.Equal(t, test.expectedMainType, test.tp.MainType)
 		require.Equal(t, test.expectedSubType, test.tp.SubType)
-		require.Equal(t, test.expectedSuffix, test.tp.Suffix)
+		require.Equal(t, test.expectedSuffix, test.tp.Suffix(), test.tp.String())
 		require.Equal(t, defaultDelimiter, test.tp.Delimiter)
 
 		require.Equal(t, test.expectedType, test.tp.Type())
@@ -61,34 +61,68 @@ func TestGetByType(t *testing.T) {
 
 	_, found = types.GetByType("text/nono")
 	require.False(t, found)
+
+	mt, found = types.GetByType("application/rss+xml")
+	require.True(t, found)
+	require.Equal(t, mt, RSSType)
+
+	mt, found = types.GetByType("application/rss")
+	require.True(t, found)
+	require.Equal(t, mt, RSSType)
+}
+
+func TestGetByMainSubType(t *testing.T) {
+	assert := require.New(t)
+	f, found := DefaultTypes.GetByMainSubType("text", "plain")
+	assert.True(found)
+	assert.Equal(f, TextType)
+	_, found = DefaultTypes.GetByMainSubType("foo", "plain")
+	assert.False(found)
 }
 
 func TestGetFirstBySuffix(t *testing.T) {
 	assert := require.New(t)
 	f, found := DefaultTypes.GetFirstBySuffix("xml")
 	assert.True(found)
-	assert.Equal(Type{MainType: "application", SubType: "rss", Suffix: "xml", Delimiter: "."}, f)
+	assert.Equal(Type{MainType: "application", SubType: "rss", OldSuffix: "xml", Delimiter: ".", Suffixes: []string{"xml"}, fileSuffix: "xml"}, f)
 }
 
 func TestFromTypeString(t *testing.T) {
-	f, err := FromString("text/html")
+	f, err := fromString("text/html")
 	require.NoError(t, err)
-	require.Equal(t, HTMLType, f)
+	require.Equal(t, HTMLType.Type(), f.Type())
 
-	f, err = FromString("application/custom")
+	f, err = fromString("application/custom")
 	require.NoError(t, err)
-	require.Equal(t, Type{MainType: "application", SubType: "custom", Suffix: "custom", Delimiter: defaultDelimiter}, f)
+	require.Equal(t, Type{MainType: "application", SubType: "custom", OldSuffix: "", fileSuffix: ""}, f)
 
-	f, err = FromString("application/custom+pdf")
+	f, err = fromString("application/custom+sfx")
 	require.NoError(t, err)
-	require.Equal(t, Type{MainType: "application", SubType: "custom", Suffix: "pdf", Delimiter: defaultDelimiter}, f)
+	require.Equal(t, Type{MainType: "application", SubType: "custom", OldSuffix: "sfx"}, f)
 
-	_, err = FromString("noslash")
+	_, err = fromString("noslash")
 	require.Error(t, err)
 
-	f, err = FromString("text/xml; charset=utf-8")
+	f, err = fromString("text/xml; charset=utf-8")
 	require.NoError(t, err)
-	require.Equal(t, Type{MainType: "text", SubType: "xml", Suffix: "xml", Delimiter: "."}, f)
+	require.Equal(t, Type{MainType: "text", SubType: "xml", OldSuffix: ""}, f)
+	require.Equal(t, "", f.Suffix())
+}
+
+// Add a test for the SVG case
+// https://github.com/gohugoio/hugo/issues/4920
+func TestFromExtensionMultipleSuffixes(t *testing.T) {
+	assert := require.New(t)
+	tp, found := DefaultTypes.GetBySuffix("svg")
+	assert.True(found)
+	assert.Equal("image/svg+xml", tp.String())
+	assert.Equal("svg", tp.fileSuffix)
+	assert.Equal(".svg", tp.FullSuffix())
+	tp, found = DefaultTypes.GetByType("image/svg+xml")
+	assert.True(found)
+	assert.Equal("image/svg+xml", tp.String())
+	assert.True(found)
+	assert.Equal(".svg", tp.FullSuffix())
 
 }
 
@@ -105,13 +139,40 @@ func TestDecodeTypes(t *testing.T) {
 			[]map[string]interface{}{
 				{
 					"application/json": map[string]interface{}{
-						"suffix": "jsn"}}},
+						"suffixes": []string{"jasn"}}}},
 			false,
 			func(t *testing.T, name string, tt Types) {
 				require.Len(t, tt, len(DefaultTypes))
-				json, found := tt.GetBySuffix("jsn")
+				json, found := tt.GetBySuffix("jasn")
 				require.True(t, found)
-				require.Equal(t, "application/json+jsn", json.String(), name)
+				require.Equal(t, "application/json", json.String(), name)
+			}},
+		{
+			"Suffix from key, multiple file suffixes",
+			[]map[string]interface{}{
+				{
+					"application/hugo+hg": map[string]interface{}{
+						"Suffixes": []string{"hg1", "hg2"},
+					}}},
+			false,
+			func(t *testing.T, name string, tt Types) {
+				require.Len(t, tt, len(DefaultTypes)+1)
+				hg, found := tt.GetBySuffix("hg")
+				require.True(t, found)
+				require.Equal(t, "hg", hg.OldSuffix)
+				require.Equal(t, "hg", hg.Suffix())
+				require.Equal(t, ".hg", hg.FullSuffix())
+				require.Equal(t, "application/hugo+hg", hg.String(), name)
+				hg, found = tt.GetBySuffix("hg2")
+				require.True(t, found)
+				require.Equal(t, "hg", hg.OldSuffix)
+				require.Equal(t, "hg2", hg.Suffix())
+				require.Equal(t, ".hg2", hg.FullSuffix())
+				require.Equal(t, "application/hugo+hg", hg.String(), name)
+
+				hg, found = tt.GetByType("application/hugo+hg")
+				require.True(t, found)
+
 			}},
 		{
 			"Add custom media type",
@@ -123,6 +184,7 @@ func TestDecodeTypes(t *testing.T) {
 			func(t *testing.T, name string, tt Types) {
 				require.Len(t, tt, len(DefaultTypes)+1)
 				// Make sure we have not broken the default config.
+
 				_, found := tt.GetBySuffix("json")
 				require.True(t, found)
 
@@ -130,15 +192,6 @@ func TestDecodeTypes(t *testing.T) {
 				require.True(t, found)
 				require.Equal(t, "text/hugo+hgo", hugo.String(), name)
 			}},
-		{
-			"Add media type invalid key",
-			[]map[string]interface{}{
-				{
-					"text/hugo+hgo": map[string]interface{}{}}},
-			true,
-			func(t *testing.T, name string, tt Types) {
-
-			}},
 	}
 
 	for _, test := range tests {