diff options
author | Ayke van Laethem <[email protected]> | 2019-11-11 15:26:15 +0100 |
---|---|---|
committer | Ron Evans <[email protected]> | 2019-11-11 20:53:50 +0100 |
commit | 8e6cb89ceb47ec031863b8edb0cc4002215acc43 (patch) | |
tree | d61699293f9f19e9e8d02b42f65b2ab631e9f768 /builder | |
parent | 946e2dd405b66eecd51a9d9f3f82555c7e13a7d7 (diff) | |
download | tinygo-8e6cb89ceb47ec031863b8edb0cc4002215acc43.tar.gz tinygo-8e6cb89ceb47ec031863b8edb0cc4002215acc43.zip |
main: refactor compile/link part to a builder package
This is a large commit that moves all code directly related to
compiling/linking into a new builder package. This has a number of
advantages:
* It cleanly separates the API between the command line and the full
compilation (with a very small API surface).
* When the compiler finally compiles one package at a time (instead of
everything at once as it does now), something will have to invoke it
once per package. This builder package will be the natural place to
do that, and also be the place where the whole process can be
parallelized.
* It allows the TinyGo compiler to be used as a package. A client can
simply import the builder package and compile code using it.
As part of this refactor, the following additional things changed:
* Exported symbols have been made unexported when they weren't needed.
* The compilation target has been moved into the compileopts.Options
struct. This is done because the target really is just another
compiler option, and the API is simplified by moving it in there.
* The moveFile function has been duplicated. It does not really belong
in the builder API but is used both by the builder and the command
line. Moving it into a separate package didn't seem useful either
for what is essentially an utility function.
* Some doc strings have been improved.
Some future changes/refactors I'd like to make after this commit:
* Clean up the API between the builder and the compiler package.
* Perhaps move the test files (in testdata/) into the builder package.
* Perhaps move the loader package into the builder package.
Diffstat (limited to 'builder')
-rw-r--r-- | builder/build.go | 227 | ||||
-rw-r--r-- | builder/buildcache.go | 118 | ||||
-rw-r--r-- | builder/builtins.go | 295 | ||||
-rw-r--r-- | builder/commands.go | 55 | ||||
-rw-r--r-- | builder/config.go | 39 | ||||
-rw-r--r-- | builder/env.go | 114 | ||||
-rw-r--r-- | builder/error.go | 25 | ||||
-rw-r--r-- | builder/linker-builtin.go | 71 | ||||
-rw-r--r-- | builder/linker-external.go | 27 | ||||
-rw-r--r-- | builder/lld.cpp | 19 | ||||
-rw-r--r-- | builder/objcopy.go | 131 | ||||
-rw-r--r-- | builder/sizes.go | 162 | ||||
-rw-r--r-- | builder/uf2.go | 130 |
13 files changed, 1413 insertions, 0 deletions
diff --git a/builder/build.go b/builder/build.go new file mode 100644 index 000000000..ceb4c0f96 --- /dev/null +++ b/builder/build.go @@ -0,0 +1,227 @@ +// Package builder is the compiler driver of TinyGo. It takes in a package name +// and an output path, and outputs an executable. It manages the entire +// compilation pipeline in between. +package builder + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/tinygo-org/tinygo/compileopts" + "github.com/tinygo-org/tinygo/compiler" + "github.com/tinygo-org/tinygo/goenv" + "github.com/tinygo-org/tinygo/interp" +) + +// Build performs a single package to executable Go build. It takes in a package +// name, an output path, and set of compile options and from that it manages the +// whole compilation process. +// +// The error value may be of type *MultiError. Callers will likely want to check +// for this case and print such errors individually. +func Build(pkgName, outpath string, config *compileopts.Config, action func(string) error) error { + c, err := compiler.NewCompiler(pkgName, config) + if err != nil { + return err + } + + // Compile Go code to IR. + errs := c.Compile(pkgName) + if len(errs) != 0 { + if len(errs) == 1 { + return errs[0] + } + return &MultiError{errs} + } + if config.Options.PrintIR { + fmt.Println("; Generated LLVM IR:") + fmt.Println(c.IR()) + } + if err := c.Verify(); err != nil { + return errors.New("verification error after IR construction") + } + + err = interp.Run(c.Module(), config.DumpSSA()) + if err != nil { + return err + } + if err := c.Verify(); err != nil { + return errors.New("verification error after interpreting runtime.initAll") + } + + if config.GOOS() != "darwin" { + c.ApplyFunctionSections() // -ffunction-sections + } + + // Browsers cannot handle external functions that have type i64 because it + // cannot be represented exactly in JavaScript (JS only has doubles). To + // keep functions interoperable, pass int64 types as pointers to + // stack-allocated values. + // Use -wasm-abi=generic to disable this behaviour. + if config.Options.WasmAbi == "js" && strings.HasPrefix(config.Triple(), "wasm") { + err := c.ExternalInt64AsPtr() + if err != nil { + return err + } + } + + // Optimization levels here are roughly the same as Clang, but probably not + // exactly. + switch config.Options.Opt { + case "none:", "0": + err = c.Optimize(0, 0, 0) // -O0 + case "1": + err = c.Optimize(1, 0, 0) // -O1 + case "2": + err = c.Optimize(2, 0, 225) // -O2 + case "s": + err = c.Optimize(2, 1, 225) // -Os + case "z": + err = c.Optimize(2, 2, 5) // -Oz, default + default: + err = errors.New("unknown optimization level: -opt=" + config.Options.Opt) + } + if err != nil { + return err + } + if err := c.Verify(); err != nil { + return errors.New("verification failure after LLVM optimization passes") + } + + // On the AVR, pointers can point either to flash or to RAM, but we don't + // know. As a temporary fix, load all global variables in RAM. + // In the future, there should be a compiler pass that determines which + // pointers are flash and which are in RAM so that pointers can have a + // correct address space parameter (address space 1 is for flash). + if strings.HasPrefix(config.Triple(), "avr") { + c.NonConstGlobals() + if err := c.Verify(); err != nil { + return errors.New("verification error after making all globals non-constant on AVR") + } + } + + // Generate output. + outext := filepath.Ext(outpath) + switch outext { + case ".o": + return c.EmitObject(outpath) + case ".bc": + return c.EmitBitcode(outpath) + case ".ll": + return c.EmitText(outpath) + default: + // Act as a compiler driver. + + // Create a temporary directory for intermediary files. + dir, err := ioutil.TempDir("", "tinygo") + if err != nil { + return err + } + defer os.RemoveAll(dir) + + // Write the object file. + objfile := filepath.Join(dir, "main.o") + err = c.EmitObject(objfile) + if err != nil { + return err + } + + // Load builtins library from the cache, possibly compiling it on the + // fly. + var librt string + if config.Target.RTLib == "compiler-rt" { + librt, err = loadBuiltins(config.Triple()) + if err != nil { + return err + } + } + + // Prepare link command. + executable := filepath.Join(dir, "main") + tmppath := executable // final file + ldflags := append(config.LDFlags(), "-o", executable, objfile) + if config.Target.RTLib == "compiler-rt" { + ldflags = append(ldflags, librt) + } + + // Compile extra files. + root := goenv.Get("TINYGOROOT") + for i, path := range config.ExtraFiles() { + abspath := filepath.Join(root, path) + outpath := filepath.Join(dir, "extra-"+strconv.Itoa(i)+"-"+filepath.Base(path)+".o") + cmdNames := []string{config.Target.Compiler} + if names, ok := commands[config.Target.Compiler]; ok { + cmdNames = names + } + err := execCommand(cmdNames, append(config.CFlags(), "-c", "-o", outpath, abspath)...) + if err != nil { + return &commandError{"failed to build", path, err} + } + ldflags = append(ldflags, outpath) + } + + // Compile C files in packages. + for i, pkg := range c.Packages() { + for _, file := range pkg.CFiles { + path := filepath.Join(pkg.Package.Dir, file) + outpath := filepath.Join(dir, "pkg"+strconv.Itoa(i)+"-"+file+".o") + cmdNames := []string{config.Target.Compiler} + if names, ok := commands[config.Target.Compiler]; ok { + cmdNames = names + } + err := execCommand(cmdNames, append(config.CFlags(), "-c", "-o", outpath, path)...) + if err != nil { + return &commandError{"failed to build", path, err} + } + ldflags = append(ldflags, outpath) + } + } + + // Link the object files together. + err = link(config.Target.Linker, ldflags...) + if err != nil { + return &commandError{"failed to link", executable, err} + } + + if config.Options.PrintSizes == "short" || config.Options.PrintSizes == "full" { + sizes, err := loadProgramSize(executable) + if err != nil { + return err + } + if config.Options.PrintSizes == "short" { + fmt.Printf(" code data bss | flash ram\n") + fmt.Printf("%7d %7d %7d | %7d %7d\n", sizes.Code, sizes.Data, sizes.BSS, sizes.Code+sizes.Data, sizes.Data+sizes.BSS) + } else { + fmt.Printf(" code rodata data bss | flash ram | package\n") + for _, name := range sizes.sortedPackageNames() { + pkgSize := sizes.Packages[name] + fmt.Printf("%7d %7d %7d %7d | %7d %7d | %s\n", pkgSize.Code, pkgSize.ROData, pkgSize.Data, pkgSize.BSS, pkgSize.Flash(), pkgSize.RAM(), name) + } + fmt.Printf("%7d %7d %7d %7d | %7d %7d | (sum)\n", sizes.Sum.Code, sizes.Sum.ROData, sizes.Sum.Data, sizes.Sum.BSS, sizes.Sum.Flash(), sizes.Sum.RAM()) + fmt.Printf("%7d - %7d %7d | %7d %7d | (all)\n", sizes.Code, sizes.Data, sizes.BSS, sizes.Code+sizes.Data, sizes.Data+sizes.BSS) + } + } + + // Get an Intel .hex file or .bin file from the .elf file. + if outext == ".hex" || outext == ".bin" || outext == ".gba" { + tmppath = filepath.Join(dir, "main"+outext) + err := objcopy(executable, tmppath) + if err != nil { + return err + } + } else if outext == ".uf2" { + // Get UF2 from the .elf file. + tmppath = filepath.Join(dir, "main"+outext) + err := convertELFFileToUF2File(executable, tmppath) + if err != nil { + return err + } + } + return action(tmppath) + } +} diff --git a/builder/buildcache.go b/builder/buildcache.go new file mode 100644 index 000000000..912cc8e09 --- /dev/null +++ b/builder/buildcache.go @@ -0,0 +1,118 @@ +package builder + +import ( + "io" + "os" + "path/filepath" + "time" + + "github.com/tinygo-org/tinygo/goenv" +) + +// Return the newest timestamp of all the file paths passed in. Used to check +// for stale caches. +func cacheTimestamp(paths []string) (time.Time, error) { + var timestamp time.Time + for _, path := range paths { + st, err := os.Stat(path) + if err != nil { + return time.Time{}, err + } + if timestamp.IsZero() { + timestamp = st.ModTime() + } else if timestamp.Before(st.ModTime()) { + timestamp = st.ModTime() + } + } + return timestamp, nil +} + +// Try to load a given file from the cache. Return "", nil if no cached file can +// be found (or the file is stale), return the absolute path if there is a cache +// and return an error on I/O errors. +// +// TODO: the configKey is currently ignored. It is supposed to be used as extra +// data for the cache key, like the compiler version and arguments. +func cacheLoad(name, configKey string, sourceFiles []string) (string, error) { + cachepath := filepath.Join(goenv.Get("GOCACHE"), name) + cacheStat, err := os.Stat(cachepath) + if os.IsNotExist(err) { + return "", nil // does not exist + } else if err != nil { + return "", err // cannot stat cache file + } + + sourceTimestamp, err := cacheTimestamp(sourceFiles) + if err != nil { + return "", err // cannot stat source files + } + + if cacheStat.ModTime().After(sourceTimestamp) { + return cachepath, nil + } else { + os.Remove(cachepath) + // stale cache + return "", nil + } +} + +// Store the file located at tmppath in the cache with the given name. The +// tmppath may or may not be gone afterwards. +// +// Note: the configKey is ignored, see cacheLoad. +func cacheStore(tmppath, name, configKey string, sourceFiles []string) (string, error) { + // get the last modified time + if len(sourceFiles) == 0 { + panic("cache: no source files") + } + + // TODO: check the config key + + dir := goenv.Get("GOCACHE") + err := os.MkdirAll(dir, 0777) + if err != nil { + return "", err + } + cachepath := filepath.Join(dir, name) + err = moveFile(tmppath, cachepath) + if err != nil { + return "", err + } + return cachepath, nil +} + +// moveFile renames the file from src to dst. If renaming doesn't work (for +// example, the rename crosses a filesystem boundary), the file is copied and +// the old file is removed. +func moveFile(src, dst string) error { + err := os.Rename(src, dst) + if err == nil { + // Success! + return nil + } + // Failed to move, probably a different filesystem. + // Do a copy + remove. + inf, err := os.Open(src) + if err != nil { + return err + } + defer inf.Close() + outpath := dst + ".tmp" + outf, err := os.Create(outpath) + if err != nil { + return err + } + + _, err = io.Copy(outf, inf) + if err != nil { + os.Remove(outpath) + return err + } + + err = outf.Close() + if err != nil { + return err + } + + return os.Rename(dst+".tmp", dst) +} diff --git a/builder/builtins.go b/builder/builtins.go new file mode 100644 index 000000000..21e0a7ae8 --- /dev/null +++ b/builder/builtins.go @@ -0,0 +1,295 @@ +package builder + +import ( + "errors" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/blakesmith/ar" + "github.com/tinygo-org/tinygo/goenv" +) + +// These are the GENERIC_SOURCES according to CMakeList.txt. +var genericBuiltins = []string{ + "absvdi2.c", + "absvsi2.c", + "absvti2.c", + "adddf3.c", + "addsf3.c", + "addtf3.c", + "addvdi3.c", + "addvsi3.c", + "addvti3.c", + "apple_versioning.c", + "ashldi3.c", + "ashlti3.c", + "ashrdi3.c", + "ashrti3.c", + "bswapdi2.c", + "bswapsi2.c", + "clzdi2.c", + "clzsi2.c", + "clzti2.c", + "cmpdi2.c", + "cmpti2.c", + "comparedf2.c", + "comparesf2.c", + "ctzdi2.c", + "ctzsi2.c", + "ctzti2.c", + "divdc3.c", + "divdf3.c", + "divdi3.c", + "divmoddi4.c", + "divmodsi4.c", + "divsc3.c", + "divsf3.c", + "divsi3.c", + "divtc3.c", + "divti3.c", + "divtf3.c", + "extendsfdf2.c", + "extendhfsf2.c", + "ffsdi2.c", + "ffssi2.c", + "ffsti2.c", + "fixdfdi.c", + "fixdfsi.c", + "fixdfti.c", + "fixsfdi.c", + "fixsfsi.c", + "fixsfti.c", + "fixunsdfdi.c", + "fixunsdfsi.c", + "fixunsdfti.c", + "fixunssfdi.c", + "fixunssfsi.c", + "fixunssfti.c", + "floatdidf.c", + "floatdisf.c", + "floatsidf.c", + "floatsisf.c", + "floattidf.c", + "floattisf.c", + "floatundidf.c", + "floatundisf.c", + "floatunsidf.c", + "floatunsisf.c", + "floatuntidf.c", + "floatuntisf.c", + //"int_util.c", + "lshrdi3.c", + "lshrti3.c", + "moddi3.c", + "modsi3.c", + "modti3.c", + "muldc3.c", + "muldf3.c", + "muldi3.c", + "mulodi4.c", + "mulosi4.c", + "muloti4.c", + "mulsc3.c", + "mulsf3.c", + "multi3.c", + "multf3.c", + "mulvdi3.c", + "mulvsi3.c", + "mulvti3.c", + "negdf2.c", + "negdi2.c", + "negsf2.c", + "negti2.c", + "negvdi2.c", + "negvsi2.c", + "negvti2.c", + "os_version_check.c", + "paritydi2.c", + "paritysi2.c", + "parityti2.c", + "popcountdi2.c", + "popcountsi2.c", + "popcountti2.c", + "powidf2.c", + "powisf2.c", + "powitf2.c", + "subdf3.c", + "subsf3.c", + "subvdi3.c", + "subvsi3.c", + "subvti3.c", + "subtf3.c", + "trampoline_setup.c", + "truncdfhf2.c", + "truncdfsf2.c", + "truncsfhf2.c", + "ucmpdi2.c", + "ucmpti2.c", + "udivdi3.c", + "udivmoddi4.c", + "udivmodsi4.c", + "udivmodti4.c", + "udivsi3.c", + "udivti3.c", + "umoddi3.c", + "umodsi3.c", + "umodti3.c", +} + +var aeabiBuiltins = []string{ + "arm/aeabi_cdcmp.S", + "arm/aeabi_cdcmpeq_check_nan.c", + "arm/aeabi_cfcmp.S", + "arm/aeabi_cfcmpeq_check_nan.c", + "arm/aeabi_dcmp.S", + "arm/aeabi_div0.c", + "arm/aeabi_drsub.c", + "arm/aeabi_fcmp.S", + "arm/aeabi_frsub.c", + "arm/aeabi_idivmod.S", + "arm/aeabi_ldivmod.S", + "arm/aeabi_memcmp.S", + "arm/aeabi_memcpy.S", + "arm/aeabi_memmove.S", + "arm/aeabi_memset.S", + "arm/aeabi_uidivmod.S", + "arm/aeabi_uldivmod.S", +} + +func builtinFiles(target string) []string { + builtins := append([]string{}, genericBuiltins...) // copy genericBuiltins + if strings.HasPrefix(target, "arm") { + builtins = append(builtins, aeabiBuiltins...) + } + return builtins +} + +// builtinsDir returns the directory where the sources for compiler-rt are kept. +func builtinsDir() string { + return filepath.Join(goenv.Get("TINYGOROOT"), "lib", "compiler-rt", "lib", "builtins") +} + +// Get the builtins archive, possibly generating it as needed. +func loadBuiltins(target string) (path string, err error) { + // Try to load a precompiled compiler-rt library. + precompiledPath := filepath.Join(goenv.Get("TINYGOROOT"), "pkg", target, "compiler-rt.a") + if _, err := os.Stat(precompiledPath); err == nil { + // Found a precompiled compiler-rt for this OS/architecture. Return the + // path directly. + return precompiledPath, nil + } + + outfile := "librt-" + target + ".a" + builtinsDir := builtinsDir() + + builtins := builtinFiles(target) + srcs := make([]string, len(builtins)) + for i, name := range builtins { + srcs[i] = filepath.Join(builtinsDir, name) + } + + if path, err := cacheLoad(outfile, commands["clang"][0], srcs); path != "" || err != nil { + return path, err + } + + var cachepath string + err = CompileBuiltins(target, func(path string) error { + path, err := cacheStore(path, outfile, commands["clang"][0], srcs) + cachepath = path + return err + }) + return cachepath, err +} + +// CompileBuiltins compiles builtins from compiler-rt into a static library. +// When it succeeds, it will call the callback with the resulting path. The path +// will be removed after callback returns. If callback returns an error, this is +// passed through to the return value of this function. +func CompileBuiltins(target string, callback func(path string) error) error { + builtinsDir := builtinsDir() + + builtins := builtinFiles(target) + srcs := make([]string, len(builtins)) + for i, name := range builtins { + srcs[i] = filepath.Join(builtinsDir, name) + } + + dirPrefix := "tinygo-builtins" + remapDir := filepath.Join(os.TempDir(), dirPrefix) + dir, err := ioutil.TempDir(os.TempDir(), dirPrefix) + if err != nil { + return err + } + defer os.RemoveAll(dir) + + // Compile all builtins. + // TODO: use builtins optimized for a given target if available. + objs := make([]string, 0, len(builtins)) + for _, name := range builtins { + objname := name + if strings.LastIndexByte(objname, '/') >= 0 { + objname = objname[strings.LastIndexByte(objname, '/'):] + } + objpath := filepath.Join(dir, objname+".o") + objs = append(objs, objpath) + srcpath := filepath.Join(builtinsDir, name) + // Note: -fdebug-prefix-map is necessary to make the output archive + // reproducible. Otherwise the temporary directory is stored in the + // archive itself, which varies each run. + err := execCommand(commands["clang"], "-c", "-Oz", "-g", "-Werror", "-Wall", "-std=c11", "-fshort-enums", "-nostdlibinc", "-ffunction-sections", "-fdata-sections", "--target="+target, "-fdebug-prefix-map="+dir+"="+remapDir, "-o", objpath, srcpath) + if err != nil { + return &commandError{"failed to build", srcpath, err} + } + } + + // Put all builtins in an archive to link as a static library. + // Note: this does not create a symbol index, but ld.lld doesn't seem to + // care. + arpath := filepath.Join(dir, "librt.a") + arfile, err := os.Create(arpath) + if err != nil { + return err + } + defer arfile.Close() + arwriter := ar.NewWriter(arfile) + err = arwriter.WriteGlobalHeader() + if err != nil { + return &os.PathError{"write ar header", arpath, err} + } + for _, objpath := range objs { + name := filepath.Base(objpath) + objfile, err := os.Open(objpath) + if err != nil { + return err + } + defer objfile.Close() + st, err := objfile.Stat() + if err != nil { + return err + } + arwriter.WriteHeader(&ar.Header{ + Name: name, + ModTime: time.Unix(0, 0), + Uid: 0, + Gid: 0, + Mode: 0644, + Size: st.Size(), + }) + n, err := io.Copy(arwriter, objfile) + if err != nil { + return err + } + if n != st.Size() { + return errors.New("file modified during ar creation: " + arpath) + } + } + + // Give the caller the resulting file. The callback must copy the file, + // because after it returns the temporary directory will be removed. + arfile.Close() + return callback(arpath) +} diff --git a/builder/commands.go b/builder/commands.go new file mode 100644 index 000000000..9ee43538c --- /dev/null +++ b/builder/commands.go @@ -0,0 +1,55 @@ +package builder + +import ( + "errors" + "os" + "os/exec" + "runtime" + "strings" +) + +// Commands lists command alternatives for various operating systems. These +// commands may have a slightly different name across operating systems and +// distributions or may not even exist in $PATH, in which case absolute paths +// may be used. +var commands = map[string][]string{ + "clang": {"clang-8"}, + "ld.lld": {"ld.lld-8", "ld.lld"}, + "wasm-ld": {"wasm-ld-8", "wasm-ld"}, +} + +func init() { + // Add the path to a Homebrew-installed LLVM 8 for ease of use (no need to + // manually set $PATH). + if runtime.GOOS == "darwin" { + commands["clang"] = append(commands["clang"], "/usr/local/opt/llvm@8/bin/clang-8") + commands["ld.lld"] = append(commands["ld.lld"], "/usr/local/opt/llvm@8/bin/ld.lld") + commands["wasm-ld"] = append(commands["wasm-ld"], "/usr/local/opt/llvm@8/bin/wasm-ld") + } + // Add the path for when LLVM was installed with the installer from + // llvm.org, which by default doesn't add LLVM to the $PATH environment + // variable. + if runtime.GOOS == "windows" { + commands["clang"] = append(commands["clang"], "clang", "C:\\Program Files\\LLVM\\bin\\clang.exe") + commands["ld.lld"] = append(commands["ld.lld"], "lld", "C:\\Program Files\\LLVM\\bin\\lld.exe") + commands["wasm-ld"] = append(commands["wasm-ld"], "C:\\Program Files\\LLVM\\bin\\wasm-ld.exe") + } +} + +func execCommand(cmdNames []string, args ...string) error { + for _, cmdName := range cmdNames { + cmd := exec.Command(cmdName, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + if err, ok := err.(*exec.Error); ok && (err.Err == exec.ErrNotFound || err.Err.Error() == "file does not exist") { + // this command was not found, try the next + continue + } + return err + } + return nil + } + return errors.New("none of these commands were found in your $PATH: " + strings.Join(cmdNames, " ")) +} diff --git a/builder/config.go b/builder/config.go new file mode 100644 index 000000000..8bb3a1162 --- /dev/null +++ b/builder/config.go @@ -0,0 +1,39 @@ +package builder + +import ( + "errors" + "fmt" + + "github.com/tinygo-org/tinygo/compileopts" + "github.com/tinygo-org/tinygo/goenv" +) + +// NewConfig builds a new Config object from a set of compiler options. It also +// loads some information from the environment while doing that. For example, it +// uses the currently active GOPATH (from the goenv package) to determine the Go +// version to use. +func NewConfig(options *compileopts.Options) (*compileopts.Config, error) { + spec, err := compileopts.LoadTarget(options.Target) + if err != nil { + return nil, err + } + goroot := goenv.Get("GOROOT") + if goroot == "" { + return nil, errors.New("cannot locate $GOROOT, please set it manually") + } + major, minor, err := getGorootVersion(goroot) + if err != nil { + return nil, fmt.Errorf("could not read version from GOROOT (%v): %v", goroot, err) + } + if major != 1 || (minor != 11 && minor != 12 && minor != 13) { + return nil, fmt.Errorf("requires go version 1.11, 1.12, or 1.13, got go%d.%d", major, minor) + } + clangHeaderPath := getClangHeaderPath(goenv.Get("TINYGOROOT")) + return &compileopts.Config{ + Options: options, + Target: spec, + GoMinorVersion: minor, + ClangHeaders: clangHeaderPath, + TestConfig: options.TestConfig, + }, nil +} diff --git a/builder/env.go b/builder/env.go new file mode 100644 index 000000000..370de728a --- /dev/null +++ b/builder/env.go @@ -0,0 +1,114 @@ +package builder + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" +) + +// getGorootVersion returns the major and minor version for a given GOROOT path. +// If the goroot cannot be determined, (0, 0) is returned. +func getGorootVersion(goroot string) (major, minor int, err error) { + s, err := GorootVersionString(goroot) + if err != nil { + return 0, 0, err + } + + if s == "" || s[:2] != "go" { + return 0, 0, errors.New("could not parse Go version: version does not start with 'go' prefix") + } + + parts := strings.Split(s[2:], ".") + if len(parts) < 2 { + return 0, 0, errors.New("could not parse Go version: version has less than two parts") + } + + // Ignore the errors, we don't really handle errors here anyway. + var trailing string + n, err := fmt.Sscanf(s, "go%d.%d%s", &major, &minor, &trailing) + if n == 2 && err == io.EOF { + // Means there were no trailing characters (i.e., not an alpha/beta) + err = nil + } + if err != nil { + return 0, 0, fmt.Errorf("failed to parse version: %s", err) + } + return +} + +// GorootVersionString returns the version string as reported by the Go +// toolchain for the given GOROOT path. It is usually of the form `go1.x.y` but +// can have some variations (for beta releases, for example). +func GorootVersionString(goroot string) (string, error) { + if data, err := ioutil.ReadFile(filepath.Join( + goroot, "src", "runtime", "internal", "sys", "zversion.go")); err == nil { + + r := regexp.MustCompile("const TheVersion = `(.*)`") + matches := r.FindSubmatch(data) + if len(matches) != 2 { + return "", errors.New("Invalid go version output:\n" + string(data)) + } + + return string(matches[1]), nil + + } else if data, err := ioutil.ReadFile(filepath.Join(goroot, "VERSION")); err == nil { + return string(data), nil + + } else { + return "", err + } +} + +// getClangHeaderPath returns the path to the built-in Clang headers. It tries +// multiple locations, which should make it find the directory when installed in +// various ways. +func getClangHeaderPath(TINYGOROOT string) string { + // Check whether we're running from the source directory. + path := filepath.Join(TINYGOROOT, "llvm", "tools", "clang", "lib", "Headers") + if _, err := os.Stat(path); !os.IsNotExist(err) { + return path + } + + // Check whether we're running from the installation directory. + path = filepath.Join(TINYGOROOT, "lib", "clang", "include") + if _, err := os.Stat(path); !os.IsNotExist(err) { + return path + } + + // It looks like we are built with a system-installed LLVM. Do a last + // attempt: try to use Clang headers relative to the clang binary. + for _, cmdName := range commands["clang"] { + binpath, err := exec.LookPath(cmdName) + if err == nil { + // This should be the command that will also be used by + // execCommand. To avoid inconsistencies, make sure we use the + // headers relative to this command. + binpath, err = filepath.EvalSymlinks(binpath) + if err != nil { + // Unexpected. + return "" + } + // Example executable: + // /usr/lib/llvm-8/bin/clang + // Example include path: + // /usr/lib/llvm-8/lib/clang/8.0.1/include/ + llvmRoot := filepath.Dir(filepath.Dir(binpath)) + clangVersionRoot := filepath.Join(llvmRoot, "lib", "clang") + dirnames, err := ioutil.ReadDir(clangVersionRoot) + if err != nil || len(dirnames) != 1 { + // Unexpected. + return "" + } + return filepath.Join(clangVersionRoot, dirnames[0].Name(), "include") + } + } + + // Could not find it. + return "" +} diff --git a/builder/error.go b/builder/error.go new file mode 100644 index 000000000..a6f59eb08 --- /dev/null +++ b/builder/error.go @@ -0,0 +1,25 @@ +package builder + +// MultiError is a list of multiple errors (actually: diagnostics) returned +// during LLVM IR generation. +type MultiError struct { + Errs []error +} + +func (e *MultiError) Error() string { + // Return the first error, to conform to the error interface. Clients should + // really do a type-assertion on *MultiError. + return e.Errs[0].Error() +} + +// commandError is an error type to wrap os/exec.Command errors. This provides +// some more information regarding what went wrong while running a command. +type commandError struct { + Msg string + File string + Err error +} + +func (e *commandError) Error() string { + return e.Msg + " " + e.File + ": " + e.Err.Error() +} diff --git a/builder/linker-builtin.go b/builder/linker-builtin.go new file mode 100644 index 000000000..ace0d2ebf --- /dev/null +++ b/builder/linker-builtin.go @@ -0,0 +1,71 @@ +// +build byollvm + +package builder + +// This file provides a Link() function that uses the bundled lld if possible. + +import ( + "errors" + "os" + "os/exec" + "unsafe" + + "github.com/tinygo-org/tinygo/goenv" +) + +/* +#include <stdbool.h> +#include <stdlib.h> +bool tinygo_link_elf(int argc, char **argv); +bool tinygo_link_wasm(int argc, char **argv); +*/ +import "C" + +// link invokes a linker with the given name and flags. +// +// This version uses the built-in linker when trying to use lld. +func link(linker string, flags ...string) error { + switch linker { + case "ld.lld": + flags = append([]string{"tinygo:" + linker}, flags...) + var cflag *C.char + buf := C.calloc(C.size_t(len(flags)), C.size_t(unsafe.Sizeof(cflag))) + cflags := (*[1 << 10]*C.char)(unsafe.Pointer(buf))[:len(flags):len(flags)] + for i, flag := range flags { + cflag := C.CString(flag) + cflags[i] = cflag + defer C.free(unsafe.Pointer(cflag)) + } + ok := C.tinygo_link_elf(C.int(len(flags)), (**C.char)(buf)) + if !ok { + return errors.New("failed to link using built-in ld.lld") + } + return nil + case "wasm-ld": + flags = append([]string{"tinygo:" + linker}, flags...) + var cflag *C.char + buf := C.calloc(C.size_t(len(flags)), C.size_t(unsafe.Sizeof(cflag))) + defer C.free(buf) + cflags := (*[1 << 10]*C.char)(unsafe.Pointer(buf))[:len(flags):len(flags)] + for i, flag := range flags { + cflag := C.CString(flag) + cflags[i] = cflag + defer C.free(unsafe.Pointer(cflag)) + } + ok := C.tinygo_link_wasm(C.int(len(flags)), (**C.char)(buf)) + if !ok { + return errors.New("failed to link using built-in wasm-ld") + } + return nil + default: + // Fall back to external command. + if cmdNames, ok := commands[linker]; ok { + return execCommand(cmdNames, flags...) + } + cmd := exec.Command(linker, flags...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Dir = goenv.Get("TINYGOROOT") + return cmd.Run() + } +} diff --git a/builder/linker-external.go b/builder/linker-external.go new file mode 100644 index 000000000..472645eb7 --- /dev/null +++ b/builder/linker-external.go @@ -0,0 +1,27 @@ +// +build !byollvm + +package builder + +// This file provides a Link() function that always runs an external command. It +// is provided for when tinygo is built without linking to liblld. + +import ( + "os" + "os/exec" + + "github.com/tinygo-org/tinygo/goenv" +) + +// link invokes a linker with the given name and arguments. +// +// This version always runs the linker as an external command. +func link(linker string, flags ...string) error { + if cmdNames, ok := commands[linker]; ok { + return execCommand(cmdNames, flags...) + } + cmd := exec.Command(linker, flags...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Dir = goenv.Get("TINYGOROOT") + return cmd.Run() +} diff --git a/builder/lld.cpp b/builder/lld.cpp new file mode 100644 index 000000000..2373edeec --- /dev/null +++ b/builder/lld.cpp @@ -0,0 +1,19 @@ +// +build byollvm + +// This file provides C wrappers for liblld. + +#include <lld/Common/Driver.h> + +extern "C" { + +bool tinygo_link_elf(int argc, char **argv) { + std::vector<const char*> args(argv, argv + argc); + return lld::elf::link(args, false); +} + +bool tinygo_link_wasm(int argc, char **argv) { + std::vector<const char*> args(argv, argv + argc); + return lld::wasm::link(args, false); +} + +} // external "C" diff --git a/builder/objcopy.go b/builder/objcopy.go new file mode 100644 index 000000000..3ab0eaeff --- /dev/null +++ b/builder/objcopy.go @@ -0,0 +1,131 @@ +package builder + +import ( + "debug/elf" + "io/ioutil" + "os" + "path/filepath" + "sort" + + "github.com/marcinbor85/gohex" +) + +// objcopyError is an error returned by functions that act like objcopy. +type objcopyError struct { + Op string + Err error +} + +func (e objcopyError) Error() string { + if e.Err == nil { + return e.Op + } + return e.Op + ": " + e.Err.Error() +} + +type progSlice []*elf.Prog + +func (s progSlice) Len() int { return len(s) } +func (s progSlice) Less(i, j int) bool { return s[i].Paddr < s[j].Paddr } +func (s progSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// extractROM extracts a firmware image and the first load address from the +// given ELF file. It tries to emulate the behavior of objcopy. +func extractROM(path string) (uint64, []byte, error) { + f, err := elf.Open(path) + if err != nil { + return 0, nil, objcopyError{"failed to open ELF file to extract text segment", err} + } + defer f.Close() + + // The GNU objcopy command does the following for firmware extraction (from + // the man page): + // > When objcopy generates a raw binary file, it will essentially produce a + // > memory dump of the contents of the input object file. All symbols and + // > relocation information will be discarded. The memory dump will start at + // > the load address of the lowest section copied into the output file. + + // Find the lowest section address. + startAddr := ^uint64(0) + for _, section := range f.Sections { + if section.Type != elf.SHT_PROGBITS || section.Flags&elf.SHF_ALLOC == 0 { + continue + } + if section.Addr < startAddr { + startAddr = section.Addr + } + } + + progs := make(progSlice, 0, 2) + for _, prog := range f.Progs { + if prog.Type != elf.PT_LOAD || prog.Filesz == 0 { + continue + } + progs = append(progs, prog) + } + if len(progs) == 0 { + return 0, nil, objcopyError{"file does not contain ROM segments: " + path, nil} + } + sort.Sort(progs) + + var rom []byte + for _, prog := range progs { + if prog.Paddr != progs[0].Paddr+uint64(len(rom)) { + return 0, nil, objcopyError{"ROM segments are non-contiguous: " + path, nil} + } + data, err := ioutil.ReadAll(prog.Open()) + if err != nil { + return 0, nil, objcopyError{"failed to extract segment from ELF file: " + path, err} + } + rom = append(rom, data...) + } + if progs[0].Paddr < startAddr { + // The lowest memory address is before the first section. This means + // that there is some extra data loaded at the start of the image that + // should be discarded. + // Example: ELF files where .text doesn't start at address 0 because + // there is a bootloader at the start. + return startAddr, rom[startAddr-progs[0].Paddr:], nil + } else { + return progs[0].Paddr, rom, nil + } +} + +// objcopy converts an ELF file to a different (simpler) output file format: +// .bin or .hex. It extracts only the .text section. +func objcopy(infile, outfile string) error { + f, err := os.OpenFile(outfile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + return err + } + defer f.Close() + + // Read the .text segment. + addr, data, err := extractROM(infile) + if err != nil { + return err + } + + // Write to the file, in the correct format. + switch filepath.Ext(outfile) { + case ".gba": + // The address is not stored in a .gba file. + _, err := f.Write(data) + return err + case ".bin": + // The address is not stored in a .bin file (therefore you + // should use .hex files in most cases). + _, err := f.Write(data) + return err + case ".hex": + mem := gohex.NewMemory() + err := mem.AddBinary(uint32(addr), data) + if err != nil { + return objcopyError{"failed to create .hex file", err} + } + mem.DumpIntelHex(f, 16) // TODO: handle error + return nil + default: + panic("unreachable") + } +} diff --git a/builder/sizes.go b/builder/sizes.go new file mode 100644 index 000000000..c88c9b7ba --- /dev/null +++ b/builder/sizes.go @@ -0,0 +1,162 @@ +package builder + +import ( + "debug/elf" + "sort" + "strings" +) + +// programSize contains size statistics per package of a compiled program. +type programSize struct { + Packages map[string]*packageSize + Sum *packageSize + Code uint64 + Data uint64 + BSS uint64 +} + +// sortedPackageNames returns the list of package names (ProgramSize.Packages) +// sorted alphabetically. +func (ps *programSize) sortedPackageNames() []string { + names := make([]string, 0, len(ps.Packages)) + for name := range ps.Packages { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// packageSize contains the size of a package, calculated from the linked object +// file. +type packageSize struct { + Code uint64 + ROData uint64 + Data uint64 + BSS uint64 +} + +// Flash usage in regular microcontrollers. +func (ps *packageSize) Flash() uint64 { + return ps.Code + ps.ROData + ps.Data +} + +// Static RAM usage in regular microcontrollers. +func (ps *packageSize) RAM() uint64 { + return ps.Data + ps.BSS +} + +type symbolList []elf.Symbol + +func (l symbolList) Len() int { + return len(l) +} + +func (l symbolList) Less(i, j int) bool { + bind_i := elf.ST_BIND(l[i].Info) + bind_j := elf.ST_BIND(l[j].Info) + if l[i].Value == l[j].Value && bind_i != elf.STB_WEAK && bind_j == elf.STB_WEAK { + // sort weak symbols after non-weak symbols + return true + } + return l[i].Value < l[j].Value +} + +func (l symbolList) Swap(i, j int) { + l[i], l[j] = l[j], l[i] +} + +// loadProgramSize calculate a program/data size breakdown of each package for a +// given ELF file. +func loadProgramSize(path string) (*programSize, error) { + file, err := elf.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + var sumCode uint64 + var sumData uint64 + var sumBSS uint64 + for _, section := range file.Sections { + if section.Flags&elf.SHF_ALLOC == 0 { + continue + } + if section.Type != elf.SHT_PROGBITS && section.Type != elf.SHT_NOBITS { + continue + } + if section.Type == elf.SHT_NOBITS { + sumBSS += section.Size + } else if section.Flags&elf.SHF_EXECINSTR != 0 { + sumCode += section.Size + } else if section.Flags&elf.SHF_WRITE != 0 { + sumData += section.Size + } + } + + allSymbols, err := file.Symbols() + if err != nil { + return nil, err + } + symbols := make([]elf.Symbol, 0, len(allSymbols)) + for _, symbol := range allSymbols { + symType := elf.ST_TYPE(symbol.Info) + if symbol.Size == 0 { + continue + } + if symType != elf.STT_FUNC && symType != elf.STT_OBJECT && symType != elf.STT_NOTYPE { + continue + } + if symbol.Section >= elf.SectionIndex(len(file.Sections)) { + continue + } + section := file.Sections[symbol.Section] + if section.Flags&elf.SHF_ALLOC == 0 { + continue + } + symbols = append(symbols, symbol) + } + sort.Sort(symbolList(symbols)) + + sizes := map[string]*packageSize{} + var lastSymbolValue uint64 + for _, symbol := range symbols { + symType := elf.ST_TYPE(symbol.Info) + //bind := elf.ST_BIND(symbol.Info) + section := file.Sections[symbol.Section] + pkgName := "(bootstrap)" + symName := strings.TrimLeft(symbol.Name, "(*") + dot := strings.IndexByte(symName, '.') + if dot > 0 { + pkgName = symName[:dot] + } + pkgSize := sizes[pkgName] + if pkgSize == nil { + pkgSize = &packageSize{} + sizes[pkgName] = pkgSize + } + if lastSymbolValue != symbol.Value || lastSymbolValue == 0 { + if symType == elf.STT_FUNC { + pkgSize.Code += symbol.Size + } else if section.Flags&elf.SHF_WRITE != 0 { + if section.Type == elf.SHT_NOBITS { + pkgSize.BSS += symbol.Size + } else { + pkgSize.Data += symbol.Size + } + } else { + pkgSize.ROData += symbol.Size + } + } + lastSymbolValue = symbol.Value + } + + sum := &packageSize{} + for _, pkg := range sizes { + sum.Code += pkg.Code + sum.ROData += pkg.ROData + sum.Data += pkg.Data + sum.BSS += pkg.BSS + } + + return &programSize{Packages: sizes, Code: sumCode, Data: sumData, BSS: sumBSS, Sum: sum}, nil +} diff --git a/builder/uf2.go b/builder/uf2.go new file mode 100644 index 000000000..3ff0a0453 --- /dev/null +++ b/builder/uf2.go @@ -0,0 +1,130 @@ +package builder + +// This file converts firmware files from BIN to UF2 format before flashing. +// +// For more information about the UF2 firmware file format, please see: +// https://github.com/Microsoft/uf2 +// +// + +import ( + "bytes" + "encoding/binary" + "io/ioutil" +) + +// convertELFFileToUF2File converts an ELF file to a UF2 file. +func convertELFFileToUF2File(infile, outfile string) error { + // Read the .text segment. + targetAddress, data, err := extractROM(infile) + if err != nil { + return err + } + + output, _ := convertBinToUF2(data, uint32(targetAddress)) + return ioutil.WriteFile(outfile, output, 0644) +} + +// convertBinToUF2 converts the binary bytes in input to UF2 formatted data. +func convertBinToUF2(input []byte, targetAddr uint32) ([]byte, int) { + blocks := split(input, 256) + output := make([]byte, 0) + + bl := newUF2Block(targetAddr) + bl.SetNumBlocks(len(blocks)) + + for i := 0; i < len(blocks); i++ { + bl.SetBlockNo(i) + bl.SetData(blocks[i]) + + output = append(output, bl.Bytes()...) + bl.IncrementAddress(bl.payloadSize) + } + + return output, len(blocks) +} + +const ( + uf2MagicStart0 = 0x0A324655 // "UF2\n" + uf2MagicStart1 = 0x9E5D5157 // Randomly selected + uf2MagicEnd = 0x0AB16F30 // Ditto +) + +// uf2Block is the structure used for each UF2 code block sent to device. +type uf2Block struct { + magicStart0 uint32 + magicStart1 uint32 + flags uint32 + targetAddr uint32 + payloadSize uint32 + blockNo uint32 + numBlocks uint32 + familyID uint32 + data []uint8 + magicEnd uint32 +} + +// newUF2Block returns a new uf2Block struct that has been correctly populated +func newUF2Block(targetAddr uint32) *uf2Block { + return &uf2Block{magicStart0: uf2MagicStart0, + magicStart1: uf2MagicStart1, + magicEnd: uf2MagicEnd, + targetAddr: targetAddr, + flags: 0x0, + familyID: 0x0, + payloadSize: 256, + data: make([]byte, 476), + } +} + +// Bytes converts the uf2Block to a slice of bytes that can be written to file. +func (b *uf2Block) Bytes() []byte { + buf := bytes.NewBuffer(make([]byte, 0, 512)) + binary.Write(buf, binary.LittleEndian, b.magicStart0) + binary.Write(buf, binary.LittleEndian, b.magicStart1) + binary.Write(buf, binary.LittleEndian, b.flags) + binary.Write(buf, binary.LittleEndian, b.targetAddr) + binary.Write(buf, binary.LittleEndian, b.payloadSize) + binary.Write(buf, binary.LittleEndian, b.blockNo) + binary.Write(buf, binary.LittleEndian, b.numBlocks) + binary.Write(buf, binary.LittleEndian, b.familyID) + binary.Write(buf, binary.LittleEndian, b.data) + binary.Write(buf, binary.LittleEndian, b.magicEnd) + + return buf.Bytes() +} + +// IncrementAddress moves the target address pointer forward by count bytes. +func (b *uf2Block) IncrementAddress(count uint32) { + b.targetAddr += b.payloadSize +} + +// SetData sets the data to be used for the current block. +func (b *uf2Block) SetData(d []byte) { + b.data = make([]byte, 476) + copy(b.data[:], d) +} + +// SetBlockNo sets the current block number to be used. +func (b *uf2Block) SetBlockNo(bn int) { + b.blockNo = uint32(bn) +} + +// SetNumBlocks sets the total number of blocks for this UF2 file. +func (b *uf2Block) SetNumBlocks(total int) { + b.numBlocks = uint32(total) +} + +// split splits a slice of bytes into a slice of byte slices of a specific size limit. +func split(input []byte, limit int) [][]byte { + var block []byte + output := make([][]byte, 0, len(input)/limit+1) + for len(input) >= limit { + block, input = input[:limit], input[limit:] + output = append(output, block) + } + if len(input) > 0 { + output = append(output, input[:len(input)]) + } + return output +} |