loader: rewrite/refactor much of the code to use go list directly

There were a few problems with the go/packages package. While it is more
or less designed for our purpose, it didn't work quite well as it didn't
provide access to indirectly imported packages (most importantly the
runtime package). This led to a workaround that sometimes broke
`tinygo test`.

This PR contains a number of related changes:

  * It uses `go list` directly to retrieve the list of packages/files to
    compile, instead of relying on the go/packages package.
  * It replaces our custom TestMain replace code with the standard code
    for running tests (generated by `go list`).
  * It adds a dummy runtime/pprof package and modifies the testing
    package, to get tests to run again with the code generated by
    `go list`.

5 files changed, 227 insertions, 253 deletions

diff --git a/loader/errors.go b/loader/errors.go
index e0414204b..5c1a2b9d7 100644
--- a/loader/errors.go
+++ b/loader/errors.go
@@ -1,5 +1,7 @@
 package loader
 
+import "go/scanner"
+
 // Errors contains a list of parser errors or a list of typechecker errors for
 // the given package.
 type Errors struct {
@@ -10,3 +12,14 @@ type Errors struct {
 func (e Errors) Error() string {
 	return "could not compile: " + e.Errs[0].Error()
 }
+
+// Error is a regular error but with an added import stack. This is especially
+// useful for debugging import cycle errors.
+type Error struct {
+	ImportStack []string
+	Err         scanner.Error
+}
+
+func (e Error) Error() string {
+	return e.Err.Error()
+}
diff --git a/loader/goroot.go b/loader/goroot.go
index c40a9f3af..964855071 100644
--- a/loader/goroot.go
+++ b/loader/goroot.go
@@ -212,7 +212,7 @@ func pathsToOverride(needsSyscallPackage bool) map[string]bool {
 		"reflect/":              false,
 		"runtime/":              false,
 		"sync/":                 true,
-		"testing/":              false,
+		"testing/":              true,
 	}
 	if needsSyscallPackage {
 		paths["syscall/"] = true // include syscall/js
diff --git a/loader/list.go b/loader/list.go
new file mode 100644
index 000000000..e33f34e4e
--- /dev/null
+++ b/loader/list.go
@@ -0,0 +1,30 @@
+package loader
+
+import (
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/tinygo-org/tinygo/compileopts"
+)
+
+// List returns a ready-to-run *exec.Cmd for running the `go list` command with
+// the configuration used for TinyGo.
+func List(config *compileopts.Config, extraArgs, pkgs []string) (*exec.Cmd, error) {
+	goroot, err := GetCachedGoroot(config)
+	if err != nil {
+		return nil, err
+	}
+	args := append([]string{"list"}, extraArgs...)
+	if len(config.BuildTags()) != 0 {
+		args = append(args, "-tags", strings.Join(config.BuildTags(), " "))
+	}
+	args = append(args, pkgs...)
+	cgoEnabled := "0"
+	if config.CgoEnabled() {
+		cgoEnabled = "1"
+	}
+	cmd := exec.Command("go", args...)
+	cmd.Env = append(os.Environ(), "GOROOT="+goroot, "GOOS="+config.GOOS(), "GOARCH="+config.GOARCH(), "CGO_ENABLED="+cgoEnabled)
+	return cmd, nil
+}
diff --git a/loader/loader.go b/loader/loader.go
index baeecc041..42d212255 100644
--- a/loader/loader.go
+++ b/loader/loader.go
@@ -2,165 +2,178 @@ package loader
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"go/ast"
-	"go/build"
 	"go/parser"
 	"go/scanner"
 	"go/token"
 	"go/types"
+	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
-	"sort"
 	"strconv"
 	"strings"
-	"text/template"
+	"syscall"
 
 	"github.com/tinygo-org/tinygo/cgo"
+	"github.com/tinygo-org/tinygo/compileopts"
 	"github.com/tinygo-org/tinygo/goenv"
-	"golang.org/x/tools/go/packages"
 )
 
 // Program holds all packages and some metadata about the program as a whole.
 type Program struct {
-	Build        *build.Context
-	Tests        bool
-	Packages     map[string]*Package
-	MainPkg      *Package
-	sorted       []*Package
-	fset         *token.FileSet
-	TypeChecker  types.Config
-	Dir          string // current working directory (for error reporting)
-	TINYGOROOT   string // root of the TinyGo installation or root of the source code
-	CFlags       []string
-	LDFlags      []string
-	ClangHeaders string
+	config       *compileopts.Config
+	clangHeaders string
+	typeChecker  types.Config
+	goroot       string // synthetic GOROOT
+	workingDir   string
+
+	Packages map[string]*Package
+	sorted   []*Package
+	fset     *token.FileSet
+
+	// Information obtained during parsing.
+	LDFlags []string
+}
+
+// PackageJSON is a subset of the JSON struct returned from `go list`.
+type PackageJSON struct {
+	Dir        string
+	ImportPath string
+	ForTest    string
+
+	// Source files
+	GoFiles  []string
+	CgoFiles []string
+	CFiles   []string
+
+	// Dependency information
+	Imports   []string
+	ImportMap map[string]string
+
+	// Error information
+	Error *struct {
+		ImportStack []string
+		Pos         string
+		Err         string
+	}
 }
 
 // Package holds a loaded package, its imports, and its parsed files.
 type Package struct {
-	*Program
-	*packages.Package
-	Files []*ast.File
-	Pkg   *types.Package
-	types.Info
+	PackageJSON
+
+	program *Program
+	Files   []*ast.File
+	Pkg     *types.Package
+	info    types.Info
 }
 
 // Load loads the given package with all dependencies (including the runtime
 // package). Call .Parse() afterwards to parse all Go files (including CGo
 // processing, if necessary).
-func (p *Program) Load(importPath string) error {
-	if p.Packages == nil {
-		p.Packages = make(map[string]*Package)
+func Load(config *compileopts.Config, inputPkgs []string, clangHeaders string, typeChecker types.Config) (*Program, error) {
+	goroot, err := GetCachedGoroot(config)
+	if err != nil {
+		return nil, err
 	}
-
-	err := p.loadPackage(importPath)
+	wd, err := os.Getwd()
 	if err != nil {
-		return err
+		return nil, err
 	}
-	p.MainPkg = p.sorted[len(p.sorted)-1]
-	if _, ok := p.Packages["runtime"]; !ok {
-		// The runtime package wasn't loaded. Although `go list -deps` seems to
-		// return the full dependency list, there is no way to get those
-		// packages from the go/packages package. Therefore load the runtime
-		// manually and add it to the list of to-be-compiled packages
-		// (duplicates are already filtered).
-		return p.loadPackage("runtime")
+	p := &Program{
+		config:       config,
+		clangHeaders: clangHeaders,
+		typeChecker:  typeChecker,
+		goroot:       goroot,
+		workingDir:   wd,
+		Packages:     make(map[string]*Package),
+		fset:         token.NewFileSet(),
 	}
-	return nil
-}
 
-func (p *Program) loadPackage(importPath string) error {
-	cgoEnabled := "0"
-	if p.Build.CgoEnabled {
-		cgoEnabled = "1"
+	// List the dependencies of this package, in raw JSON format.
+	extraArgs := []string{"-json", "-deps"}
+	if config.TestConfig.CompileTestBinary {
+		extraArgs = append(extraArgs, "-test")
 	}
-	pkgs, err := packages.Load(&packages.Config{
-		Mode:       packages.NeedName | packages.NeedFiles | packages.NeedImports | packages.NeedDeps,
-		Env:        append(os.Environ(), "GOROOT="+p.Build.GOROOT, "GOOS="+p.Build.GOOS, "GOARCH="+p.Build.GOARCH, "CGO_ENABLED="+cgoEnabled),
-		BuildFlags: []string{"-tags", strings.Join(p.Build.BuildTags, " ")},
-		Tests:      p.Tests,
-	}, importPath)
+	cmd, err := List(config, extraArgs, inputPkgs)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	var pkg *packages.Package
-	if p.Tests {
-		// We need the second package. Quoting from the docs:
-		// > For example, when using the go command, loading "fmt" with Tests=true
-		// > returns four packages, with IDs "fmt" (the standard package),
-		// > "fmt [fmt.test]" (the package as compiled for the test),
-		// > "fmt_test" (the test functions from source files in package fmt_test),
-		// > and "fmt.test" (the test binary).
-		pkg = pkgs[1]
-	} else {
-		if len(pkgs) != 1 {
-			return fmt.Errorf("expected exactly one package while importing %s, got %d", importPath, len(pkgs))
+	buf := &bytes.Buffer{}
+	cmd.Stdout = buf
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
+				os.Exit(status.ExitStatus())
+			}
+			os.Exit(1)
 		}
-		pkg = pkgs[0]
-	}
-	var importError *Errors
-	var addPackages func(pkg *packages.Package)
-	addPackages = func(pkg *packages.Package) {
-		if _, ok := p.Packages[pkg.PkgPath]; ok {
-			return
+		return nil, fmt.Errorf("failed to run `go list`: %s", err)
+	}
+
+	// Parse the returned json from `go list`.
+	decoder := json.NewDecoder(buf)
+	for {
+		pkg := &Package{
+			program: p,
+			info: types.Info{
+				Types:      make(map[ast.Expr]types.TypeAndValue),
+				Defs:       make(map[*ast.Ident]types.Object),
+				Uses:       make(map[*ast.Ident]types.Object),
+				Implicits:  make(map[ast.Node]types.Object),
+				Scopes:     make(map[ast.Node]*types.Scope),
+				Selections: make(map[*ast.SelectorExpr]*types.Selection),
+			},
 		}
-		pkg2 := p.newPackage(pkg)
-		p.Packages[pkg.PkgPath] = pkg2
-		if len(pkg.Errors) != 0 {
-			if importError != nil {
-				// There was another error reported already. Do not report
-				// errors from multiple packages at once.
-				return
+		err := decoder.Decode(&pkg.PackageJSON)
+		if err != nil {
+			if err == io.EOF {
+				break
 			}
-			importError = &Errors{
-				Pkg: pkg2,
+			return nil, err
+		}
+		if pkg.Error != nil {
+			// There was an error while importing (for example, a circular
+			// dependency).
+			pos := token.Position{}
+			fields := strings.Split(pkg.Error.Pos, ":")
+			if len(fields) >= 2 {
+				// There is some file/line/column information.
+				if n, err := strconv.Atoi(fields[len(fields)-2]); err == nil {
+					// Format: filename.go:line:colum
+					pos.Filename = strings.Join(fields[:len(fields)-2], ":")
+					pos.Line = n
+					pos.Column, _ = strconv.Atoi(fields[len(fields)-1])
+				} else {
+					// Format: filename.go:line
+					pos.Filename = strings.Join(fields[:len(fields)-1], ":")
+					pos.Line, _ = strconv.Atoi(fields[len(fields)-1])
+				}
+				pos.Filename = p.getOriginalPath(pos.Filename)
 			}
-			for _, err := range pkg.Errors {
-				pos := token.Position{}
-				fields := strings.Split(err.Pos, ":")
-				if len(fields) >= 2 {
-					// There is some file/line/column information.
-					if n, err := strconv.Atoi(fields[len(fields)-2]); err == nil {
-						// Format: filename.go:line:colum
-						pos.Filename = strings.Join(fields[:len(fields)-2], ":")
-						pos.Line = n
-						pos.Column, _ = strconv.Atoi(fields[len(fields)-1])
-					} else {
-						// Format: filename.go:line
-						pos.Filename = strings.Join(fields[:len(fields)-1], ":")
-						pos.Line, _ = strconv.Atoi(fields[len(fields)-1])
-					}
-					pos.Filename = p.getOriginalPath(pos.Filename)
+			err := scanner.Error{
+				Pos: pos,
+				Msg: pkg.Error.Err,
+			}
+			if len(pkg.Error.ImportStack) != 0 {
+				return nil, Error{
+					ImportStack: pkg.Error.ImportStack,
+					Err:         err,
 				}
-				importError.Errs = append(importError.Errs, scanner.Error{
-					Pos: pos,
-					Msg: err.Msg,
-				})
 			}
-			return
-		}
-
-		// Get the list of imports (sorted alphabetically).
-		names := make([]string, 0, len(pkg.Imports))
-		for name := range pkg.Imports {
-			names = append(names, name)
-		}
-		sort.Strings(names)
-
-		// Add all the imports.
-		for _, name := range names {
-			addPackages(pkg.Imports[name])
+			return nil, err
 		}
-
-		p.sorted = append(p.sorted, pkg2)
-	}
-	addPackages(pkg)
-	if importError != nil {
-		return *importError
+		p.sorted = append(p.sorted, pkg)
+		p.Packages[pkg.ImportPath] = pkg
 	}
-	return nil
+
+	return p, nil
 }
 
 // getOriginalPath looks whether this path is in the generated GOROOT and if so,
@@ -168,23 +181,23 @@ func (p *Program) loadPackage(importPath string) error {
 // the input path is returned.
 func (p *Program) getOriginalPath(path string) string {
 	originalPath := path
-	if strings.HasPrefix(path, p.Build.GOROOT+string(filepath.Separator)) {
+	if strings.HasPrefix(path, p.goroot+string(filepath.Separator)) {
 		// If this file is part of the synthetic GOROOT, try to infer the
 		// original path.
-		relpath := path[len(filepath.Join(p.Build.GOROOT, "src"))+1:]
+		relpath := path[len(filepath.Join(p.goroot, "src"))+1:]
 		realgorootPath := filepath.Join(goenv.Get("GOROOT"), "src", relpath)
 		if _, err := os.Stat(realgorootPath); err == nil {
 			originalPath = realgorootPath
 		}
 		maybeInTinyGoRoot := false
-		for prefix := range pathsToOverride(needsSyscallPackage(p.Build.BuildTags)) {
+		for prefix := range pathsToOverride(needsSyscallPackage(p.config.BuildTags())) {
 			if !strings.HasPrefix(relpath, prefix) {
 				continue
 			}
 			maybeInTinyGoRoot = true
 		}
 		if maybeInTinyGoRoot {
-			tinygoPath := filepath.Join(p.TINYGOROOT, "src", relpath)
+			tinygoPath := filepath.Join(goenv.Get("TINYGOROOT"), "src", relpath)
 			if _, err := os.Stat(tinygoPath); err == nil {
 				originalPath = tinygoPath
 			}
@@ -193,28 +206,18 @@ func (p *Program) getOriginalPath(path string) string {
 	return originalPath
 }
 
-// newPackage instantiates a new *Package object with initialized members.
-func (p *Program) newPackage(pkg *packages.Package) *Package {
-	return &Package{
-		Program: p,
-		Package: pkg,
-		Info: types.Info{
-			Types:      make(map[ast.Expr]types.TypeAndValue),
-			Defs:       make(map[*ast.Ident]types.Object),
-			Uses:       make(map[*ast.Ident]types.Object),
-			Implicits:  make(map[ast.Node]types.Object),
-			Scopes:     make(map[ast.Node]*types.Scope),
-			Selections: make(map[*ast.SelectorExpr]*types.Selection),
-		},
-	}
-}
-
 // Sorted returns a list of all packages, sorted in a way that no packages come
 // before the packages they depend upon.
 func (p *Program) Sorted() []*Package {
 	return p.sorted
 }
 
+// MainPkg returns the last package in the Sorted() slice. This is the main
+// package of the program.
+func (p *Program) MainPkg() *Package {
+	return p.sorted[len(p.sorted)-1]
+}
+
 // Parse parses all packages and typechecks them.
 //
 // The returned error may be an Errors error, which contains a list of errors.
@@ -222,22 +225,16 @@ func (p *Program) Sorted() []*Package {
 // Idempotent.
 func (p *Program) Parse() error {
 	// Parse all packages.
-	for _, pkg := range p.Sorted() {
+	// TODO: do this in parallel.
+	for _, pkg := range p.sorted {
 		err := pkg.Parse()
 		if err != nil {
 			return err
 		}
 	}
 
-	if p.Tests {
-		err := p.swapTestMain()
-		if err != nil {
-			return err
-		}
-	}
-
 	// Typecheck all packages.
-	for _, pkg := range p.Sorted() {
+	for _, pkg := range p.sorted {
 		err := pkg.Check()
 		if err != nil {
 			return err
@@ -247,82 +244,6 @@ func (p *Program) Parse() error {
 	return nil
 }
 
-func (p *Program) swapTestMain() error {
-	var tests []string
-
-	isTestFunc := func(f *ast.FuncDecl) bool {
-		// TODO: improve signature check
-		if strings.HasPrefix(f.Name.Name, "Test") && f.Name.Name != "TestMain" {
-			return true
-		}
-		return false
-	}
-	for _, f := range p.MainPkg.Files {
-		for i, d := range f.Decls {
-			switch v := d.(type) {
-			case *ast.FuncDecl:
-				if isTestFunc(v) {
-					tests = append(tests, v.Name.Name)
-				}
-				if v.Name.Name == "main" {
-					// Remove main
-					if len(f.Decls) == 1 {
-						f.Decls = make([]ast.Decl, 0)
-					} else {
-						f.Decls[i] = f.Decls[len(f.Decls)-1]
-						f.Decls = f.Decls[:len(f.Decls)-1]
-					}
-				}
-			}
-		}
-	}
-
-	// TODO: Check if they defined a TestMain and call it instead of testing.TestMain
-	const mainBody = `package main
-
-import (
-	"testing"
-)
-
-func main () {
-	m := &testing.M{
-		Tests: []testing.TestToCall{
-{{range .TestFunctions}}
-			{Name: "{{.}}", Func: {{.}}},
-{{end}}
-		},
-	}
-
-	testing.TestMain(m)
-}
-`
-	tmpl := template.Must(template.New("testmain").Parse(mainBody))
-	b := bytes.Buffer{}
-	tmplData := struct {
-		TestFunctions []string
-	}{
-		TestFunctions: tests,
-	}
-
-	err := tmpl.Execute(&b, tmplData)
-	if err != nil {
-		return err
-	}
-	path := filepath.Join(p.MainPkg.Dir, "$testmain.go")
-
-	if p.fset == nil {
-		p.fset = token.NewFileSet()
-	}
-
-	newMain, err := parser.ParseFile(p.fset, path, b.Bytes(), parser.AllErrors)
-	if err != nil {
-		return err
-	}
-	p.MainPkg.Files = append(p.MainPkg.Files, newMain)
-
-	return nil
-}
-
 // parseFile is a wrapper around parser.ParseFile.
 func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) {
 	if p.fset == nil {
@@ -342,14 +263,13 @@ func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) {
 // Idempotent.
 func (p *Package) Parse() error {
 	if len(p.Files) != 0 {
-		return nil
+		return nil // nothing to do (?)
 	}
 
 	// Load the AST.
-	// TODO: do this in parallel.
-	if p.PkgPath == "unsafe" {
-		// Special case for the unsafe package. Don't even bother loading
-		// the files.
+	if p.ImportPath == "unsafe" {
+		// Special case for the unsafe package, which is defined internally by
+		// the types package.
 		p.Pkg = types.Unsafe
 		return nil
 	}
@@ -369,11 +289,11 @@ func (p *Package) Parse() error {
 // Idempotent.
 func (p *Package) Check() error {
 	if p.Pkg != nil {
-		return nil
+		return nil // already typechecked
 	}
 
 	var typeErrors []error
-	checker := p.TypeChecker
+	checker := p.program.typeChecker // make a copy, because it will be modified
 	checker.Error = func(err error) {
 		typeErrors = append(typeErrors, err)
 	}
@@ -381,7 +301,7 @@ func (p *Package) Check() error {
 	// Do typechecking of the package.
 	checker.Importer = p
 
-	typesPkg, err := checker.Check(p.PkgPath, p.fset, p.Files, &p.Info)
+	typesPkg, err := checker.Check(p.ImportPath, p.program.fset, p.Files, &p.info)
 	if err != nil {
 		if err, ok := err.(Errors); ok {
 			return err
@@ -394,40 +314,46 @@ func (p *Package) Check() error {
 
 // parseFiles parses the loaded list of files and returns this list.
 func (p *Package) parseFiles() ([]*ast.File, error) {
-	// TODO: do this concurrently.
 	var files []*ast.File
 	var fileErrs []error
 
-	var cgoFiles []*ast.File
-	for _, file := range p.GoFiles {
-		f, err := p.parseFile(file, parser.ParseComments)
-		if err != nil {
-			fileErrs = append(fileErrs, err)
-			continue
+	// Parse all files (incuding CgoFiles).
+	parseFile := func(file string) {
+		if !filepath.IsAbs(file) {
+			file = filepath.Join(p.Dir, file)
 		}
+		f, err := p.program.parseFile(file, parser.ParseComments)
 		if err != nil {
 			fileErrs = append(fileErrs, err)
-			continue
-		}
-		for _, importSpec := range f.Imports {
-			if importSpec.Path.Value == `"C"` {
-				cgoFiles = append(cgoFiles, f)
-			}
+			return
 		}
 		files = append(files, f)
 	}
-	if len(cgoFiles) != 0 {
-		cflags := append(p.CFlags, "-I"+filepath.Dir(p.GoFiles[0]))
-		if p.ClangHeaders != "" {
-			cflags = append(cflags, "-Xclang", "-internal-isystem", "-Xclang", p.ClangHeaders)
+	for _, file := range p.GoFiles {
+		parseFile(file)
+	}
+	for _, file := range p.CgoFiles {
+		parseFile(file)
+	}
+
+	// Do CGo processing.
+	if len(p.CgoFiles) != 0 {
+		var cflags []string
+		cflags = append(cflags, p.program.config.CFlags()...)
+		cflags = append(cflags, "-I"+p.Dir)
+		if p.program.clangHeaders != "" {
+			cflags = append(cflags, "-Xclang", "-internal-isystem", "-Xclang", p.program.clangHeaders)
 		}
-		generated, ldflags, errs := cgo.Process(files, p.Program.Dir, p.fset, cflags)
+		generated, ldflags, errs := cgo.Process(files, p.program.workingDir, p.program.fset, cflags)
 		if errs != nil {
 			fileErrs = append(fileErrs, errs...)
 		}
 		files = append(files, generated)
-		p.LDFlags = append(p.LDFlags, ldflags...)
+		p.program.LDFlags = append(p.program.LDFlags, ldflags...)
 	}
+
+	// Only return an error after CGo processing, so that errors in parsing and
+	// CGo can be reported together.
 	if len(fileErrs) != 0 {
 		return nil, Errors{p, fileErrs}
 	}
@@ -441,8 +367,13 @@ func (p *Package) Import(to string) (*types.Package, error) {
 	if to == "unsafe" {
 		return types.Unsafe, nil
 	}
-	if _, ok := p.Imports[to]; ok {
-		return p.Packages[p.Imports[to].PkgPath].Pkg, nil
+	if replace, ok := p.ImportMap[to]; ok {
+		// This import path should be replaced by another import path, according
+		// to `go list`.
+		to = replace
+	}
+	if imported, ok := p.program.Packages[to]; ok {
+		return imported.Pkg, nil
 	} else {
 		return nil, errors.New("package not imported: " + to)
 	}
diff --git a/loader/ssa.go b/loader/ssa.go
index a047d35d6..c3b87c495 100644
--- a/loader/ssa.go
+++ b/loader/ssa.go
@@ -10,8 +10,8 @@ import (
 func (p *Program) LoadSSA() *ssa.Program {
 	prog := ssa.NewProgram(p.fset, ssa.SanityCheckFunctions|ssa.BareInits|ssa.GlobalDebug)
 
-	for _, pkg := range p.Sorted() {
-		prog.CreatePackage(pkg.Pkg, pkg.Files, &pkg.Info, true)
+	for _, pkg := range p.sorted {
+		prog.CreatePackage(pkg.Pkg, pkg.Files, &pkg.info, true)
 	}
 
 	return prog