main: implement -ldflags="-X ..."

This commit implements replacing some global variables with a different
value, if the global variable has no initializer. For example, if you
have:

    package main

    var version string

you can replace the value with -ldflags="-X main.version=0.2".

Right now it only works for uninitialized globals. The Go tooling also
supports initialized globals (var version = "<undefined>") but that is a
bit hard to combine with how initialized globals are currently
implemented.

The current implementation still allows caching package IR files while
making sure the values don't end up in the build cache. This means
compiling a program multiple times with different values will use the
cached package each time, inserting the string value only late in the
build process.

Fixes #1045

6 files changed, 196 insertions, 21 deletions

diff --git a/builder/build.go b/builder/build.go
index 319474619..f16916ceb 100644
--- a/builder/build.go
+++ b/builder/build.go
@@ -52,16 +52,17 @@ type BuildResult struct {
 // key, avoiding the need for recompiling all dependencies when only the
 // implementation of an imported package changes.
 type packageAction struct {
-	ImportPath      string
-	CompilerVersion int // compiler.Version
-	InterpVersion   int // interp.Version
-	LLVMVersion     string
-	Config          *compiler.Config
-	CFlags          []string
-	FileHashes      map[string]string // hash of every file that's part of the package
-	Imports         map[string]string // map from imported package to action ID hash
-	OptLevel        int               // LLVM optimization level (0-3)
-	SizeLevel       int               // LLVM optimization for size level (0-2)
+	ImportPath       string
+	CompilerVersion  int // compiler.Version
+	InterpVersion    int // interp.Version
+	LLVMVersion      string
+	Config           *compiler.Config
+	CFlags           []string
+	FileHashes       map[string]string // hash of every file that's part of the package
+	Imports          map[string]string // map from imported package to action ID hash
+	OptLevel         int               // LLVM optimization level (0-3)
+	SizeLevel        int               // LLVM optimization for size level (0-2)
+	UndefinedGlobals []string          // globals that are left as external globals (no initializer)
 }
 
 // Build performs a single package to executable Go build. It takes in a package
@@ -133,19 +134,26 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	for _, pkg := range lprogram.Sorted() {
 		pkg := pkg // necessary to avoid a race condition
 
+		var undefinedGlobals []string
+		for name := range config.Options.GlobalValues[pkg.Pkg.Path()] {
+			undefinedGlobals = append(undefinedGlobals, name)
+		}
+		sort.Strings(undefinedGlobals)
+
 		// Create a cache key: a hash from the action ID below that contains all
 		// the parameters for the build.
 		actionID := packageAction{
-			ImportPath:      pkg.ImportPath,
-			CompilerVersion: compiler.Version,
-			InterpVersion:   interp.Version,
-			LLVMVersion:     llvm.Version,
-			Config:          compilerConfig,
-			CFlags:          pkg.CFlags,
-			FileHashes:      make(map[string]string, len(pkg.FileHashes)),
-			Imports:         make(map[string]string, len(pkg.Pkg.Imports())),
-			OptLevel:        optLevel,
-			SizeLevel:       sizeLevel,
+			ImportPath:       pkg.ImportPath,
+			CompilerVersion:  compiler.Version,
+			InterpVersion:    interp.Version,
+			LLVMVersion:      llvm.Version,
+			Config:           compilerConfig,
+			CFlags:           pkg.CFlags,
+			FileHashes:       make(map[string]string, len(pkg.FileHashes)),
+			Imports:          make(map[string]string, len(pkg.Pkg.Imports())),
+			OptLevel:         optLevel,
+			SizeLevel:        sizeLevel,
+			UndefinedGlobals: undefinedGlobals,
 		}
 		for filePath, hash := range pkg.FileHashes {
 			actionID.FileHashes[filePath] = hex.EncodeToString(hash)
@@ -196,6 +204,25 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 					return errors.New("verification error after compiling package " + pkg.ImportPath)
 				}
 
+				// Erase all globals that are part of the undefinedGlobals list.
+				// This list comes from the -ldflags="-X pkg.foo=val" option.
+				// Instead of setting the value directly in the AST (which would
+				// mean the value, which may be a secret, is stored in the build
+				// cache), the global itself is left external (undefined) and is
+				// only set at the end of the compilation.
+				for _, name := range undefinedGlobals {
+					globalName := pkg.Pkg.Path() + "." + name
+					global := mod.NamedGlobal(globalName)
+					if global.IsNil() {
+						return errors.New("global not found: " + globalName)
+					}
+					name := global.Name()
+					newGlobal := llvm.AddGlobal(mod, global.Type().ElementType(), name+".tmp")
+					global.ReplaceAllUsesWith(newGlobal)
+					global.EraseFromParentAsGlobal()
+					newGlobal.SetName(name)
+				}
+
 				// Try to interpret package initializers at compile time.
 				// It may only be possible to do this partially, in which case
 				// it is completed after all IR files are linked.
@@ -640,6 +667,12 @@ func optimizeProgram(mod llvm.Module, config *compileopts.Config) error {
 		transform.ApplyFunctionSections(mod) // -ffunction-sections
 	}
 
+	// Insert values from -ldflags="-X ..." into the IR.
+	err = setGlobalValues(mod, config.Options.GlobalValues)
+	if err != nil {
+		return err
+	}
+
 	// Browsers cannot handle external functions that have type i64 because it
 	// cannot be represented exactly in JavaScript (JS only has doubles). To
 	// keep functions interoperable, pass int64 types as pointers to
@@ -677,6 +710,71 @@ func optimizeProgram(mod llvm.Module, config *compileopts.Config) error {
 	return nil
 }
 
+// setGlobalValues sets the global values from the -ldflags="-X ..." compiler
+// option in the given module. An error may be returned if the global is not of
+// the expected type.
+func setGlobalValues(mod llvm.Module, globals map[string]map[string]string) error {
+	var pkgPaths []string
+	for pkgPath := range globals {
+		pkgPaths = append(pkgPaths, pkgPath)
+	}
+	sort.Strings(pkgPaths)
+	for _, pkgPath := range pkgPaths {
+		pkg := globals[pkgPath]
+		var names []string
+		for name := range pkg {
+			names = append(names, name)
+		}
+		sort.Strings(names)
+		for _, name := range names {
+			value := pkg[name]
+			globalName := pkgPath + "." + name
+			global := mod.NamedGlobal(globalName)
+			if global.IsNil() || !global.Initializer().IsNil() {
+				// The global either does not exist (optimized away?) or has
+				// some value, in which case it has already been initialized at
+				// package init time.
+				continue
+			}
+
+			// A strin is a {ptr, len} pair. We need these types to build the
+			// initializer.
+			initializerType := global.Type().ElementType()
+			if initializerType.TypeKind() != llvm.StructTypeKind || initializerType.StructName() == "" {
+				return fmt.Errorf("%s: not a string", globalName)
+			}
+			elementTypes := initializerType.StructElementTypes()
+			if len(elementTypes) != 2 {
+				return fmt.Errorf("%s: not a string", globalName)
+			}
+
+			// Create a buffer for the string contents.
+			bufInitializer := mod.Context().ConstString(value, false)
+			buf := llvm.AddGlobal(mod, bufInitializer.Type(), ".string")
+			buf.SetInitializer(bufInitializer)
+			buf.SetAlignment(1)
+			buf.SetUnnamedAddr(true)
+			buf.SetLinkage(llvm.PrivateLinkage)
+
+			// Create the string value, which is a {ptr, len} pair.
+			zero := llvm.ConstInt(mod.Context().Int32Type(), 0, false)
+			ptr := llvm.ConstGEP(buf, []llvm.Value{zero, zero})
+			if ptr.Type() != elementTypes[0] {
+				return fmt.Errorf("%s: not a string", globalName)
+			}
+			length := llvm.ConstInt(elementTypes[1], uint64(len(value)), false)
+			initializer := llvm.ConstNamedStruct(initializerType, []llvm.Value{
+				ptr,
+				length,
+			})
+
+			// Set the initializer. No initializer should be set at this point.
+			global.SetInitializer(initializer)
+		}
+	}
+	return nil
+}
+
 // functionStackSizes keeps stack size information about a single function
 // (usually a goroutine).
 type functionStackSize struct {
diff --git a/compileopts/options.go b/compileopts/options.go
index a12a92cd9..7570fefe5 100644
--- a/compileopts/options.go
+++ b/compileopts/options.go
@@ -30,6 +30,7 @@ type Options struct {
 	PrintStacks   bool
 	Tags          string
 	WasmAbi       string
+	GlobalValues  map[string]map[string]string // map[pkgpath]map[varname]value
 	TestConfig    TestConfig
 	Programmer    string
 }
diff --git a/main.go b/main.go
index 0a8f06868..3d9bd6e75 100644
--- a/main.go
+++ b/main.go
@@ -18,6 +18,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/google/shlex"
 	"github.com/mattn/go-colorable"
 	"github.com/tinygo-org/tinygo/builder"
 	"github.com/tinygo-org/tinygo/compileopts"
@@ -835,6 +836,52 @@ func handleCompilerError(err error) {
 	}
 }
 
+// This is a special type for the -X flag to parse the pkgpath.Var=stringVal
+// format. It has to be a special type to allow multiple variables to be defined
+// this way.
+type globalValuesFlag map[string]map[string]string
+
+func (m globalValuesFlag) String() string {
+	return "pkgpath.Var=value"
+}
+
+func (m globalValuesFlag) Set(value string) error {
+	equalsIndex := strings.IndexByte(value, '=')
+	if equalsIndex < 0 {
+		return errors.New("expected format pkgpath.Var=value")
+	}
+	pathAndName := value[:equalsIndex]
+	pointIndex := strings.LastIndexByte(pathAndName, '.')
+	if pointIndex < 0 {
+		return errors.New("expected format pkgpath.Var=value")
+	}
+	path := pathAndName[:pointIndex]
+	name := pathAndName[pointIndex+1:]
+	stringValue := value[equalsIndex+1:]
+	if m[path] == nil {
+		m[path] = make(map[string]string)
+	}
+	m[path][name] = stringValue
+	return nil
+}
+
+// parseGoLinkFlag parses the -ldflags parameter. Its primary purpose right now
+// is the -X flag, for setting the value of global string variables.
+func parseGoLinkFlag(flagsString string) (map[string]map[string]string, error) {
+	set := flag.NewFlagSet("link", flag.ExitOnError)
+	globalVarValues := make(globalValuesFlag)
+	set.Var(globalVarValues, "X", "Set the value of the string variable to the given value.")
+	flags, err := shlex.Split(flagsString)
+	if err != nil {
+		return nil, err
+	}
+	err = set.Parse(flags)
+	if err != nil {
+		return nil, err
+	}
+	return map[string]map[string]string(globalVarValues), nil
+}
+
 func main() {
 	if len(os.Args) < 2 {
 		fmt.Fprintln(os.Stderr, "No command-line arguments supplied.")
@@ -859,6 +906,7 @@ func main() {
 	ocdOutput := flag.Bool("ocd-output", false, "print OCD daemon output during debug")
 	port := flag.String("port", "", "flash port (can specify multiple candidates separated by commas)")
 	programmer := flag.String("programmer", "", "which hardware programmer to use")
+	ldflags := flag.String("ldflags", "", "Go link tool compatible ldflags")
 	wasmAbi := flag.String("wasm-abi", "", "WebAssembly ABI conventions: js (no i64 params) or generic")
 
 	var flagJSON, flagDeps *bool
@@ -888,6 +936,11 @@ func main() {
 	}
 
 	flag.CommandLine.Parse(os.Args[2:])
+	globalVarValues, err := parseGoLinkFlag(*ldflags)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
 	options := &compileopts.Options{
 		Target:        *target,
 		Opt:           *opt,
@@ -902,13 +955,14 @@ func main() {
 		PrintStacks:   *printStacks,
 		PrintCommands: *printCommands,
 		Tags:          *tags,
+		GlobalValues:  globalVarValues,
 		WasmAbi:       *wasmAbi,
 		Programmer:    *programmer,
 	}
 
 	os.Setenv("CC", "clang -target="+*target)
 
-	err := options.Verify()
+	err = options.Verify()
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err.Error())
 		usage()
diff --git a/main_test.go b/main_test.go
index 9f6d777cb..69fe99e67 100644
--- a/main_test.go
+++ b/main_test.go
@@ -138,6 +138,18 @@ func TestCompiler(t *testing.T) {
 				Opt: "0",
 			}, nil)
 		})
+
+		t.Run("ldflags", func(t *testing.T) {
+			t.Parallel()
+			runTestWithConfig("ldflags.go", "", t, &compileopts.Options{
+				Opt: "z",
+				GlobalValues: map[string]map[string]string{
+					"main": {
+						"someGlobal": "foobar",
+					},
+				},
+			}, nil)
+		})
 	})
 }
 
diff --git a/testdata/ldflags.go b/testdata/ldflags.go
new file mode 100644
index 000000000..94db0dcb1
--- /dev/null
+++ b/testdata/ldflags.go
@@ -0,0 +1,9 @@
+package main
+
+// These globals can be changed using -ldflags="-X main.someGlobal=value".
+// At the moment, only globals without an initializer can be replaced this way.
+var someGlobal string
+
+func main() {
+	println("someGlobal:", someGlobal)
+}
diff --git a/testdata/ldflags.txt b/testdata/ldflags.txt
new file mode 100644
index 000000000..0f39abf05
--- /dev/null
+++ b/testdata/ldflags.txt
@@ -0,0 +1 @@
+someGlobal: foobar