diff options
| -rw-r--r-- | builder/build.go | 42 | ||||
| -rw-r--r-- | builder/cc.go | 4 | ||||
| -rw-r--r-- | builder/library.go | 41 | ||||
| -rw-r--r-- | go.mod | 1 | ||||
| -rw-r--r-- | go.sum | 2 |
5 files changed, 68 insertions, 22 deletions
diff --git a/builder/build.go b/builder/build.go index 3bb685a80..84efa27a3 100644 --- a/builder/build.go +++ b/builder/build.go @@ -23,6 +23,7 @@ import ( "strconv" "strings" + "github.com/gofrs/flock" "github.com/tinygo-org/tinygo/cgo" "github.com/tinygo-org/tinygo/compileopts" "github.com/tinygo-org/tinygo/compiler" @@ -97,17 +98,19 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil var libcDependencies []*compileJob switch config.Target.Libc { case "musl": - job, err := Musl.load(config, dir) + job, unlock, err := Musl.load(config, dir) if err != nil { return err } + defer unlock() libcDependencies = append(libcDependencies, dummyCompileJob(filepath.Join(filepath.Dir(job.result), "crt1.o"))) libcDependencies = append(libcDependencies, job) case "picolibc": - libcJob, err := Picolibc.load(config, dir) + libcJob, unlock, err := Picolibc.load(config, dir) if err != nil { return err } + defer unlock() libcDependencies = append(libcDependencies, libcJob) case "wasi-libc": path := filepath.Join(root, "lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a") @@ -116,10 +119,11 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil } libcDependencies = append(libcDependencies, dummyCompileJob(path)) case "mingw-w64": - _, err := MinGW.load(config, dir) + _, unlock, err := MinGW.load(config, dir) if err != nil { return err } + unlock() libcDependencies = append(libcDependencies, makeMinGWExtraLibs(dir)...) case "": // no library specified, so nothing to do @@ -228,17 +232,19 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil bitcodePath := filepath.Join(cacheDir, "pkg-"+hex.EncodeToString(hash[:])+".bc") packageBitcodePaths[pkg.ImportPath] = bitcodePath - // Check whether this package has been compiled before, and if so don't - // compile it again. - if _, err := os.Stat(bitcodePath); err == nil { - // Already cached, don't recreate this package. - continue - } - // The package has not yet been compiled, so create a job to do so. job := &compileJob{ description: "compile package " + pkg.ImportPath, run: func(*compileJob) error { + // Acquire a lock (if supported). + unlock := lock(bitcodePath + ".lock") + defer unlock() + + if _, err := os.Stat(bitcodePath); err == nil { + // Already cached, don't recreate this package. + return nil + } + // Compile AST to IR. The compiler.CompilePackage function will // build the SSA as needed. mod, errs := compiler.CompilePackage(pkg.ImportPath, pkg, program.Package(pkg.Pkg), machine, compilerConfig, config.DumpSSA()) @@ -533,10 +539,11 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil // Add compiler-rt dependency if needed. Usually this is a simple load from // a cache. if config.Target.RTLib == "compiler-rt" { - job, err := CompilerRT.load(config, dir) + job, unlock, err := CompilerRT.load(config, dir) if err != nil { return err } + defer unlock() linkerDependencies = append(linkerDependencies, job) } @@ -1181,3 +1188,16 @@ func patchRP2040BootCRC(executable string) error { // Update the .boot2 section to included the CRC return replaceElfSection(executable, ".boot2", bytes) } + +// lock may acquire a lock at the specified path. +// It returns a function to release the lock. +// If flock is not supported, it does nothing. +func lock(path string) func() { + flock := flock.New(path) + err := flock.Lock() + if err != nil { + return func() {} + } + + return func() { flock.Close() } +} diff --git a/builder/cc.go b/builder/cc.go index 3aac75586..350982d00 100644 --- a/builder/cc.go +++ b/builder/cc.go @@ -63,6 +63,10 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands return "", err } + // Acquire a lock (if supported). + unlock := lock(filepath.Join(goenv.Get("GOCACHE"), fileHash+".c.lock")) + defer unlock() + // Create cache key for the dependencies file. buf, err := json.Marshal(struct { Path string diff --git a/builder/library.go b/builder/library.go index efcc8b75a..64bd8ecd6 100644 --- a/builder/library.go +++ b/builder/library.go @@ -6,6 +6,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "github.com/tinygo-org/tinygo/compileopts" "github.com/tinygo-org/tinygo/goenv" @@ -37,10 +38,11 @@ type Library struct { // The resulting directory may be stored in the provided tmpdir, which is // expected to be removed after the Load call. func (l *Library) Load(config *compileopts.Config, tmpdir string) (dir string, err error) { - job, err := l.load(config, tmpdir) + job, unlock, err := l.load(config, tmpdir) if err != nil { return "", err } + defer unlock() err = runJobs(job, config.Options.Parallelism) return filepath.Dir(job.result), err } @@ -53,28 +55,38 @@ func (l *Library) Load(config *compileopts.Config, tmpdir string) (dir string, e // output archive file, it is expected to be removed after use. // As a side effect, this call creates the library header files if they didn't // exist yet. -func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJob, err error) { +func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJob, abortLock func(), err error) { outdir, precompiled := config.LibcPath(l.name) archiveFilePath := filepath.Join(outdir, "lib.a") if precompiled { // Found a precompiled library for this OS/architecture. Return the path // directly. - return dummyCompileJob(archiveFilePath), nil + return dummyCompileJob(archiveFilePath), func() {}, nil } + // Create a lock on the output (if supported). + // This is a bit messy, but avoids a deadlock because it is ordered consistently with other library loads within a build. + outname := filepath.Base(outdir) + unlock := lock(filepath.Join(goenv.Get("GOCACHE"), outname+".lock")) + var ok bool + defer func() { + if !ok { + unlock() + } + }() + // Try to fetch this library from the cache. if _, err := os.Stat(archiveFilePath); err == nil { - return dummyCompileJob(archiveFilePath), nil + return dummyCompileJob(archiveFilePath), func() {}, nil } // Cache miss, build it now. // Create the destination directory where the components of this library // (lib.a file, include directory) are placed. - outname := filepath.Base(outdir) err = os.MkdirAll(filepath.Join(goenv.Get("GOCACHE"), outname), 0o777) if err != nil { // Could not create directory (and not because it already exists). - return nil, err + return nil, nil, err } // Make headers if needed. @@ -84,12 +96,12 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ if _, err = os.Stat(headerPath); err != nil { temporaryHeaderPath, err := ioutil.TempDir(outdir, "include.tmp*") if err != nil { - return nil, err + return nil, nil, err } defer os.RemoveAll(temporaryHeaderPath) err = l.makeHeaders(target, temporaryHeaderPath) if err != nil { - return nil, err + return nil, nil, err } err = os.Rename(temporaryHeaderPath, headerPath) if err != nil { @@ -108,7 +120,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ fallthrough default: - return nil, err + return nil, nil, err } } } @@ -118,7 +130,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ dir := filepath.Join(tmpdir, "build-lib-"+l.name) err = os.Mkdir(dir, 0777) if err != nil { - return nil, err + return nil, nil, err } // Precalculate the flags to the compiler invocation. @@ -146,6 +158,8 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ args = append(args, "-march=rv64gc", "-mabi=lp64") } + var once sync.Once + // Create job to put all the object files in a single archive. This archive // file is the (static) library file. var objs []string @@ -153,6 +167,8 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ description: "ar " + l.name + "/lib.a", result: filepath.Join(goenv.Get("GOCACHE"), outname, "lib.a"), run: func(*compileJob) error { + defer once.Do(unlock) + // Create an archive of all object files. f, err := ioutil.TempFile(outdir, "libc.a.tmp*") if err != nil { @@ -224,5 +240,8 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ }) } - return job, nil + ok = true + return job, func() { + once.Do(unlock) + }, nil } @@ -7,6 +7,7 @@ require ( github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 github.com/chromedp/cdproto v0.0.0-20210113043257-dabd2f2e7693 github.com/chromedp/chromedp v0.6.4 + github.com/gofrs/flock v0.8.1 // indirect github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf github.com/marcinbor85/gohex v0.0.0-20200531091804-343a4b548892 github.com/mattn/go-colorable v0.1.8 @@ -17,6 +17,8 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs= github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg= github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= |