Add directory based archetypes

Given this content:

```bash
archetypes
├── default.md
└── post-bundle
    ├── bio.md
    ├── images
    │   └── featured.jpg
    └── index.md
```

```bash
hugo new --kind post-bundle post/my-post
```

Will create a new folder in `/content/post/my-post` with the same set of files as in the `post-bundle` archetypes folder.

This commit also improves the archetype language detection, so, if you use template code in your content files, the `.Site` you get is for the correct language. This also means that it is now possible to translate strings defined in  the `i18n` bundles,  e.g. `{{ i18n "hello" }}`.

Fixes #4535

3 files changed, 321 insertions, 96 deletions

diff --git a/create/content.go b/create/content.go
index 6d022282e..00924941f 100644
--- a/create/content.go
+++ b/create/content.go
@@ -17,69 +17,74 @@ package create
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
+
+	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugolib"
+	"github.com/spf13/afero"
 	jww "github.com/spf13/jwalterweatherman"
 )
 
 // NewContent creates a new content file in the content directory based upon the
 // given kind, which is used to lookup an archetype.
 func NewContent(
-	ps *helpers.PathSpec,
-	siteFactory func(filename string, siteUsed bool) (*hugolib.Site, error), kind, targetPath string) error {
+	sites *hugolib.HugoSites, kind, targetPath string) error {
+	targetPath = filepath.Clean(targetPath)
 	ext := helpers.Ext(targetPath)
-	fs := ps.BaseFs.SourceFilesystems.Archetypes.Fs
+	ps := sites.PathSpec
+	archetypeFs := ps.BaseFs.SourceFilesystems.Archetypes.Fs
+	sourceFs := ps.Fs.Source
 
 	jww.INFO.Printf("attempting to create %q of %q of ext %q", targetPath, kind, ext)
 
-	archetypeFilename := findArchetype(ps, kind, ext)
+	archetypeFilename, isDir := findArchetype(ps, kind, ext)
+	contentPath, s := resolveContentPath(sites, sourceFs, targetPath)
 
-	// Building the sites can be expensive, so only do it if really needed.
-	siteUsed := false
+	if isDir {
 
-	if archetypeFilename != "" {
-		f, err := fs.Open(archetypeFilename)
+		langFs := hugofs.NewLanguageFs(s.Language.Lang, sites.LanguageSet(), archetypeFs)
+
+		cm, err := mapArcheTypeDir(ps, langFs, archetypeFilename)
 		if err != nil {
-			return fmt.Errorf("failed to open archetype file: %s", err)
+			return err
 		}
-		defer f.Close()
 
-		if helpers.ReaderContains(f, []byte(".Site")) {
-			siteUsed = true
+		if cm.siteUsed {
+			if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
+				return err
+			}
 		}
-	}
 
-	s, err := siteFactory(targetPath, siteUsed)
-	if err != nil {
-		return err
+		name := filepath.Base(targetPath)
+		return newContentFromDir(archetypeFilename, sites, archetypeFs, sourceFs, cm, name, contentPath)
 	}
 
-	var content []byte
+	// Building the sites can be expensive, so only do it if really needed.
+	siteUsed := false
 
-	content, err = executeArcheTypeAsTemplate(s, kind, targetPath, archetypeFilename)
-	if err != nil {
-		return err
+	if archetypeFilename != "" {
+		var err error
+		siteUsed, err = usesSiteVar(archetypeFs, archetypeFilename)
+		if err != nil {
+			return err
+		}
 	}
 
-	// The site may have multiple content dirs, and we currently do not know which contentDir the
-	// user wants to create this content in. We should improve on this, but we start by testing if the
-	// provided path points to an existing dir. If so, use it as is.
-	var contentPath string
-	var exists bool
-	targetDir := filepath.Dir(targetPath)
-
-	if targetDir != "" && targetDir != "." {
-		exists, _ = helpers.Exists(targetDir, fs)
+	if siteUsed {
+		if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
+			return err
+		}
 	}
 
-	if exists {
-		contentPath = targetPath
-	} else {
-		contentPath = s.PathSpec.AbsPathify(filepath.Join(s.Cfg.GetString("contentDir"), targetPath))
+	content, err := executeArcheTypeAsTemplate(s, "", kind, targetPath, archetypeFilename)
+	if err != nil {
+		return err
 	}
 
 	if err := helpers.SafeWriteToDisk(contentPath, bytes.NewReader(content), s.Fs.Source); err != nil {
@@ -103,29 +108,199 @@ func NewContent(
 	return nil
 }
 
+func targetSite(sites *hugolib.HugoSites, fi *hugofs.LanguageFileInfo) *hugolib.Site {
+	for _, s := range sites.Sites {
+		if fi.Lang() == s.Language.Lang {
+			return s
+		}
+	}
+	return sites.Sites[0]
+}
+
+func newContentFromDir(
+	archetypeDir string,
+	sites *hugolib.HugoSites,
+	sourceFs, targetFs afero.Fs,
+	cm archetypeMap, name, targetPath string) error {
+
+	for _, f := range cm.otherFiles {
+		filename := f.Filename()
+		// Just copy the file to destination.
+		in, err := sourceFs.Open(filename)
+		if err != nil {
+			return err
+		}
+
+		targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir))
+
+		targetDir := filepath.Dir(targetFilename)
+		if err := targetFs.MkdirAll(targetDir, 0777); err != nil && !os.IsExist(err) {
+			return fmt.Errorf("failed to create target directory for %s: %s", targetDir, err)
+		}
+
+		out, err := targetFs.Create(targetFilename)
+
+		_, err = io.Copy(out, in)
+		if err != nil {
+			return err
+		}
+
+		in.Close()
+		out.Close()
+	}
+
+	for _, f := range cm.contentFiles {
+		filename := f.Filename()
+		s := targetSite(sites, f)
+		targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir))
+
+		content, err := executeArcheTypeAsTemplate(s, name, archetypeDir, targetFilename, filename)
+		if err != nil {
+			return err
+		}
+
+		if err := helpers.SafeWriteToDisk(targetFilename, bytes.NewReader(content), targetFs); err != nil {
+			return err
+		}
+	}
+
+	jww.FEEDBACK.Println(targetPath, "created")
+
+	return nil
+}
+
+type archetypeMap struct {
+	// These needs to be parsed and executed as Go templates.
+	contentFiles []*hugofs.LanguageFileInfo
+	// These are just copied to destination.
+	otherFiles []*hugofs.LanguageFileInfo
+	// If the templates needs a fully built site. This can potentially be
+	// expensive, so only do when needed.
+	siteUsed bool
+}
+
+func mapArcheTypeDir(
+	ps *helpers.PathSpec,
+	fs afero.Fs,
+	archetypeDir string) (archetypeMap, error) {
+
+	var m archetypeMap
+
+	walkFn := func(filename string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if fi.IsDir() {
+			return nil
+		}
+
+		fil := fi.(*hugofs.LanguageFileInfo)
+
+		if hugolib.IsContentFile(filename) {
+			m.contentFiles = append(m.contentFiles, fil)
+			if !m.siteUsed {
+				m.siteUsed, err = usesSiteVar(fs, filename)
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+
+		m.otherFiles = append(m.otherFiles, fil)
+
+		return nil
+	}
+
+	if err := helpers.SymbolicWalk(fs, archetypeDir, walkFn); err != nil {
+		return m, err
+	}
+
+	return m, nil
+}
+
+func usesSiteVar(fs afero.Fs, filename string) (bool, error) {
+	f, err := fs.Open(filename)
+	if err != nil {
+		return false, fmt.Errorf("failed to open archetype file: %s", err)
+	}
+	defer f.Close()
+	return helpers.ReaderContains(f, []byte(".Site")), nil
+}
+
+// Resolve the target content path.
+func resolveContentPath(sites *hugolib.HugoSites, fs afero.Fs, targetPath string) (string, *hugolib.Site) {
+	targetDir := filepath.Dir(targetPath)
+	first := sites.Sites[0]
+
+	var (
+		s              *hugolib.Site
+		siteContentDir string
+	)
+
+	// Try the filename: my-post.en.md
+	for _, ss := range sites.Sites {
+		if strings.Contains(targetPath, "."+ss.Language.Lang+".") {
+			s = ss
+			break
+		}
+	}
+
+	for _, ss := range sites.Sites {
+		contentDir := ss.PathSpec.ContentDir
+		if !strings.HasSuffix(contentDir, helpers.FilePathSeparator) {
+			contentDir += helpers.FilePathSeparator
+		}
+		if strings.HasPrefix(targetPath, contentDir) {
+			siteContentDir = ss.PathSpec.ContentDir
+			if s == nil {
+				s = ss
+			}
+			break
+		}
+	}
+
+	if s == nil {
+		s = first
+	}
+
+	if targetDir != "" && targetDir != "." {
+		exists, _ := helpers.Exists(targetDir, fs)
+
+		if exists {
+			return targetPath, s
+		}
+	}
+
+	if siteContentDir != "" {
+		pp := filepath.Join(siteContentDir, strings.TrimPrefix(targetPath, siteContentDir))
+		return s.PathSpec.AbsPathify(pp), s
+
+	} else {
+		return s.PathSpec.AbsPathify(filepath.Join(first.PathSpec.ContentDir, targetPath)), s
+	}
+
+}
+
 // FindArchetype takes a given kind/archetype of content and returns the path
 // to the archetype in the archetype filesystem, blank if none found.
-func findArchetype(ps *helpers.PathSpec, kind, ext string) (outpath string) {
+func findArchetype(ps *helpers.PathSpec, kind, ext string) (outpath string, isDir bool) {
 	fs := ps.BaseFs.Archetypes.Fs
 
-	// If the new content isn't in a subdirectory, kind == "".
-	// Therefore it should be excluded otherwise `is a directory`
-	// error will occur. github.com/gohugoio/hugo/issues/411
-	var pathsToCheck = []string{"default"}
+	var pathsToCheck []string
 
-	if ext != "" {
-		if kind != "" {
-			pathsToCheck = append([]string{kind + ext, "default" + ext}, pathsToCheck...)
-		} else {
-			pathsToCheck = append([]string{"default" + ext}, pathsToCheck...)
-		}
+	if kind != "" {
+		pathsToCheck = append(pathsToCheck, kind+ext)
 	}
+	pathsToCheck = append(pathsToCheck, "default"+ext, "default")
 
 	for _, p := range pathsToCheck {
-		if exists, _ := helpers.Exists(p, fs); exists {
-			return p
+		fi, err := fs.Stat(p)
+		if err == nil {
+			return p, fi.IsDir()
 		}
 	}
 
-	return ""
+	return "", false
 }
diff --git a/create/content_template_handler.go b/create/content_template_handler.go
index 02598d4d3..458b7285c 100644
--- a/create/content_template_handler.go
+++ b/create/content_template_handler.go
@@ -80,7 +80,7 @@ var (
 		"%}x}", "%}}")
 )
 
-func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFilename string) ([]byte, error) {
+func executeArcheTypeAsTemplate(s *hugolib.Site, name, kind, targetPath, archetypeFilename string) ([]byte, error) {
 
 	var (
 		archetypeContent  []byte
@@ -88,20 +88,16 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile
 		err               error
 	)
 
-	ps, err := helpers.NewPathSpec(s.Deps.Fs, s.Deps.Cfg)
-	if err != nil {
-		return nil, err
-	}
-	sp := source.NewSourceSpec(ps, ps.Fs.Source)
-
-	f := sp.NewFileInfo("", targetPath, false, nil)
+	f := s.SourceSpec.NewFileInfo("", targetPath, false, nil)
 
-	name := f.TranslationBaseName()
+	if name == "" {
+		name = f.TranslationBaseName()
 
-	if name == "index" || name == "_index" {
-		// Page bundles; the directory name will hopefully have a better name.
-		dir := strings.TrimSuffix(f.Dir(), helpers.FilePathSeparator)
-		_, name = filepath.Split(dir)
+		if name == "index" || name == "_index" {
+			// Page bundles; the directory name will hopefully have a better name.
+			dir := strings.TrimSuffix(f.Dir(), helpers.FilePathSeparator)
+			_, name = filepath.Split(dir)
+		}
 	}
 
 	data := ArchetypeFileData{
diff --git a/create/content_test.go b/create/content_test.go
index f3bcc1dd5..503c9da8d 100644
--- a/create/content_test.go
+++ b/create/content_test.go
@@ -35,8 +35,7 @@ import (
 )
 
 func TestNewContent(t *testing.T) {
-	v := viper.New()
-	initViper(v)
+	assert := require.New(t)
 
 	cases := []struct {
 		kind     string
@@ -49,6 +48,14 @@ func TestNewContent(t *testing.T) {
 		{"stump", "stump/sample-2.md", []string{`title: "Sample 2"`}},      // no archetype file
 		{"", "sample-3.md", []string{`title: "Sample 3"`}},                 // no archetype
 		{"product", "product/sample-4.md", []string{`title = "SAMPLE-4"`}}, // empty archetype front matter
+		{"lang", "post/lang-1.md", []string{`Site Lang: en|Name: Lang 1|i18n: Hugo Rocks!`}},
+		{"lang", "post/lang-2.en.md", []string{`Site Lang: en|Name: Lang 2|i18n: Hugo Rocks!`}},
+		{"lang", "post/lang-3.nn.md", []string{`Site Lang: nn|Name: Lang 3|i18n: Hugo Rokkar!`}},
+		{"lang", "content_nn/post/lang-4.md", []string{`Site Lang: nn|Name: Lang 4|i18n: Hugo Rokkar!`}},
+		{"lang", "content_nn/post/lang-5.en.md", []string{`Site Lang: en|Name: Lang 5|i18n: Hugo Rocks!`}},
+		{"lang", "post/my-bundle/index.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}},
+		{"lang", "post/my-bundle/index.en.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}},
+		{"lang", "post/my-bundle/index.nn.md", []string{`Site Lang: nn|Name: My Bundle|i18n: Hugo Rokkar!`}},
 		{"shortcodes", "shortcodes/go.md", []string{
 			`title = "GO"`,
 			"{{< myshortcode >}}",
@@ -56,21 +63,20 @@ func TestNewContent(t *testing.T) {
 			"{{</* comment */>}}\n{{%/* comment */%}}"}}, // shortcodes
 	}
 
-	for _, c := range cases {
-		cfg, fs := newTestCfg()
-		require.NoError(t, initFs(fs))
+	for i, c := range cases {
+		cfg, fs := newTestCfg(assert)
+		assert.NoError(initFs(fs))
 		h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs})
-		require.NoError(t, err)
+		assert.NoError(err)
 
-		siteFactory := func(filename string, siteUsed bool) (*hugolib.Site, error) {
-			return h.Sites[0], nil
-		}
-
-		require.NoError(t, create.NewContent(h.PathSpec, siteFactory, c.kind, c.path))
+		assert.NoError(create.NewContent(h, c.kind, c.path))
 
-		fname := filepath.Join("content", filepath.FromSlash(c.path))
+		fname := filepath.FromSlash(c.path)
+		if !strings.HasPrefix(fname, "content") {
+			fname = filepath.Join("content", fname)
+		}
 		content := readFileFromFs(t, fs.Source, fname)
-		for i, v := range c.expected {
+		for _, v := range c.expected {
 			found := strings.Contains(content, v)
 			if !found {
 				t.Fatalf("[%d] %q missing from output:\n%q", i, v, content)
@@ -79,17 +85,44 @@ func TestNewContent(t *testing.T) {
 	}
 }
 
-func initViper(v *viper.Viper) {
-	v.Set("metaDataFormat", "toml")
-	v.Set("archetypeDir", "archetypes")
-	v.Set("contentDir", "content")
-	v.Set("themesDir", "themes")
-	v.Set("layoutDir", "layouts")
-	v.Set("i18nDir", "i18n")
-	v.Set("theme", "sample")
-	v.Set("archetypeDir", "archetypes")
-	v.Set("resourceDir", "resources")
-	v.Set("publishDir", "public")
+func TestNewContentFromDir(t *testing.T) {
+	assert := require.New(t)
+	cfg, fs := newTestCfg(assert)
+	assert.NoError(initFs(fs))
+
+	archetypeDir := filepath.Join("archetypes", "my-bundle")
+	assert.NoError(fs.Source.Mkdir(archetypeDir, 0755))
+
+	contentFile := `
+File: %s
+Site Lang: {{ .Site.Language.Lang  }} 	
+Name: {{ replace .Name "-" " " | title }}
+i18n: {{ T "hugo" }}
+`
+
+	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755))
+	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0755))
+
+	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0755))
+	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755))
+	assert.NoError(afero.WriteFile(fs.Source, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0755))
+
+	h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs})
+	assert.NoError(err)
+	assert.Equal(2, len(h.Sites))
+
+	assert.NoError(create.NewContent(h, "my-bundle", "post/my-post"))
+
+	assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`)
+	assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo2.xml")), `hugo2: {{ printf "no template handling in here" }}`)
+
+	// Content files should get the correct site context.
+	// TODO(bep) archetype check i18n
+	assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.md")), `File: index.md`, `Site Lang: en`, `Name: My Post`, `i18n: Hugo Rocks!`)
+	assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.nn.md")), `File: index.nn.md`, `Site Lang: nn`, `Name: My Post`, `i18n: Hugo Rokkar!`)
+
+	assertContains(assert, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/pages/bio.md")), `File: bio.md`, `Site Lang: en`, `Name: My Post`)
+
 }
 
 func initFs(fs *hugofs.Fs) error {
@@ -132,6 +165,10 @@ title = "{{ .BaseFileName  | upper }}"
 			path:    filepath.Join("archetypes", "emptydate.md"),
 			content: "+++\ndate =\"\"\ntitle = \"Empty Date Arch title\"\ntest = \"test1\"\n+++\n",
 		},
+		{
+			path:    filepath.Join("archetypes", "lang.md"),
+			content: `Site Lang: {{ .Site.Language.Lang  }}|Name: {{ replace .Name "-" " " | title }}|i18n: {{ T "hugo" }}`,
+		},
 		// #3623x
 		{
 			path: filepath.Join("archetypes", "shortcodes.md"),
@@ -166,6 +203,12 @@ Some text.
 	return nil
 }
 
+func assertContains(assert *require.Assertions, v interface{}, matches ...string) {
+	for _, m := range matches {
+		assert.Contains(v, m)
+	}
+}
+
 // TODO(bep) extract common testing package with this and some others
 func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string {
 	filename = filepath.FromSlash(filename)
@@ -185,22 +228,33 @@ func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string {
 	return string(b)
 }
 
-func newTestCfg() (*viper.Viper, *hugofs.Fs) {
+func newTestCfg(assert *require.Assertions) (*viper.Viper, *hugofs.Fs) {
+
+	cfg := `
+	
+[languages]
+[languages.en]
+weight = 1
+languageName = "English"
+[languages.nn]
+weight = 2
+languageName = "Nynorsk"
+contentDir = "content_nn"
+
+`
 
-	v := viper.New()
-	v.Set("contentDir", "content")
-	v.Set("dataDir", "data")
-	v.Set("i18nDir", "i18n")
-	v.Set("layoutDir", "layouts")
-	v.Set("archetypeDir", "archetypes")
-	v.Set("assetDir", "assets")
+	mm := afero.NewMemMapFs()
 
-	fs := hugofs.NewMem(v)
+	assert.NoError(afero.WriteFile(mm, filepath.Join("i18n", "en.toml"), []byte(`[hugo]
+other = "Hugo Rocks!"`), 0755))
+	assert.NoError(afero.WriteFile(mm, filepath.Join("i18n", "nn.toml"), []byte(`[hugo]
+other = "Hugo Rokkar!"`), 0755))
 
-	v.SetFs(fs.Source)
+	assert.NoError(afero.WriteFile(mm, "config.toml", []byte(cfg), 0755))
 
-	initViper(v)
+	v, _, err := hugolib.LoadConfig(hugolib.ConfigSourceDescriptor{Fs: mm, Filename: "config.toml"})
+	assert.NoError(err)
 
-	return v, fs
+	return v, hugofs.NewFrom(mm, v)
 
 }