builder: run function passes per package

This should result in a small compile time reduction for incremental
builds, somewhere around 5-9%.

This commit, while small, required many previous commits to not regress
binary size. Right now binary size is basically identical with very few
changes in size (the only baremetal program that changed in size did so
with a 4 byte increase).

This commit is one extra step towards doing as much work as possible in
the parallel and cached package build step, out of the serial LTO phase.
Later improvements in this area have this change as a prerequisite.

2 files changed, 29 insertions, 11 deletions

diff --git a/builder/build.go b/builder/build.go
index 73981e71e..319474619 100644
--- a/builder/build.go
+++ b/builder/build.go
@@ -60,6 +60,7 @@ type packageAction struct {
 	CFlags          []string
 	FileHashes      map[string]string // hash of every file that's part of the package
 	Imports         map[string]string // map from imported package to action ID hash
+	OptLevel        int               // LLVM optimization level (0-3)
 	SizeLevel       int               // LLVM optimization for size level (0-2)
 }
 
@@ -128,7 +129,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	var packageJobs []*compileJob
 	packageBitcodePaths := make(map[string]string)
 	packageActionIDs := make(map[string]string)
-	_, sizeLevel, _ := config.OptLevels()
+	optLevel, sizeLevel, _ := config.OptLevels()
 	for _, pkg := range lprogram.Sorted() {
 		pkg := pkg // necessary to avoid a race condition
 
@@ -143,6 +144,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 			CFlags:          pkg.CFlags,
 			FileHashes:      make(map[string]string, len(pkg.FileHashes)),
 			Imports:         make(map[string]string, len(pkg.Pkg.Imports())),
+			OptLevel:        optLevel,
 			SizeLevel:       sizeLevel,
 		}
 		for filePath, hash := range pkg.FileHashes {
@@ -219,6 +221,28 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 					}
 				}
 
+				// Run function passes for each function in the module.
+				// These passes are intended to be run on each function right
+				// after they're created to reduce IR size (and maybe also for
+				// cache locality to improve performance), but for now they're
+				// run here for each function in turn. Maybe this can be
+				// improved in the future.
+				builder := llvm.NewPassManagerBuilder()
+				defer builder.Dispose()
+				builder.SetOptLevel(optLevel)
+				builder.SetSizeLevel(sizeLevel)
+				funcPasses := llvm.NewFunctionPassManagerForModule(mod)
+				defer funcPasses.Dispose()
+				builder.PopulateFunc(funcPasses)
+				funcPasses.InitializeFunc()
+				for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) {
+					if fn.IsDeclaration() {
+						continue
+					}
+					funcPasses.RunFunc(fn)
+				}
+				funcPasses.FinalizeFunc()
+
 				// Serialize the LLVM module as a bitcode file.
 				// Write to a temporary path that is renamed to the destination
 				// file to avoid race conditions with other TinyGo invocatiosn
diff --git a/transform/optimizer.go b/transform/optimizer.go
index 108219106..051f929cb 100644
--- a/transform/optimizer.go
+++ b/transform/optimizer.go
@@ -50,16 +50,6 @@ func Optimize(mod llvm.Module, config *compileopts.Config, optLevel, sizeLevel i
 		}
 	}
 
-	// Run function passes for each function.
-	funcPasses := llvm.NewFunctionPassManagerForModule(mod)
-	defer funcPasses.Dispose()
-	builder.PopulateFunc(funcPasses)
-	funcPasses.InitializeFunc()
-	for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) {
-		funcPasses.RunFunc(fn)
-	}
-	funcPasses.FinalizeFunc()
-
 	if optLevel > 0 {
 		// Run some preparatory passes for the Go optimizer.
 		goPasses := llvm.NewPassManager()
@@ -164,6 +154,10 @@ func Optimize(mod llvm.Module, config *compileopts.Config, optLevel, sizeLevel i
 
 	// Run function passes again, because without it, llvm.coro.size.i32()
 	// doesn't get lowered.
+	funcPasses := llvm.NewFunctionPassManagerForModule(mod)
+	defer funcPasses.Dispose()
+	builder.PopulateFunc(funcPasses)
+	funcPasses.InitializeFunc()
 	for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) {
 		funcPasses.RunFunc(fn)
 	}