aboutsummaryrefslogtreecommitdiffhomepage
path: root/loader/goroot.go
blob: 5442df9b5df5f4e412c82ec9d4b1bf4d6af9e9ff (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
package loader

// This file constructs a new temporary GOROOT directory by merging both the
// standard Go GOROOT and the GOROOT from TinyGo using symlinks.
//
// The goal is to replace specific packages from Go with a TinyGo version. It's
// never a partial replacement, either a package is fully replaced or it is not.
// This is important because if we did allow to merge packages (e.g. by adding
// files to a package), it would lead to a dependency on implementation details
// with all the maintenance burden that results in. Only allowing to replace
// packages as a whole avoids this as packages are already designed to have a
// public (backwards-compatible) API.

import (
	"crypto/sha512"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io"
	"io/fs"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"sort"
	"sync"

	"github.com/tinygo-org/tinygo/compileopts"
	"github.com/tinygo-org/tinygo/goenv"
)

var gorootCreateMutex sync.Mutex

// GetCachedGoroot creates a new GOROOT by merging both the standard GOROOT and
// the GOROOT from TinyGo using lots of symbolic links.
func GetCachedGoroot(config *compileopts.Config) (string, error) {
	goroot := goenv.Get("GOROOT")
	if goroot == "" {
		return "", errors.New("could not determine GOROOT")
	}
	tinygoroot := goenv.Get("TINYGOROOT")
	if tinygoroot == "" {
		return "", errors.New("could not determine TINYGOROOT")
	}

	// Find the overrides needed for the goroot.
	overrides := pathsToOverride(config.GoMinorVersion, needsSyscallPackage(config.BuildTags()))

	// Resolve the merge links within the goroot.
	merge, err := listGorootMergeLinks(goroot, tinygoroot, overrides)
	if err != nil {
		return "", err
	}

	// Hash the merge links to create a cache key.
	data, err := json.Marshal(merge)
	if err != nil {
		return "", err
	}
	hash := sha512.Sum512_256(data)

	// Do not try to create the cached GOROOT in parallel, that's only a waste
	// of I/O bandwidth and thus speed. Instead, use a mutex to make sure only
	// one goroutine does it at a time.
	// This is not a way to ensure atomicity (a different TinyGo invocation
	// could be creating the same directory), but instead a way to avoid
	// creating it many times in parallel when running tests in parallel.
	gorootCreateMutex.Lock()
	defer gorootCreateMutex.Unlock()

	// Check if the goroot already exists.
	cachedGorootName := "goroot-" + hex.EncodeToString(hash[:])
	cachedgoroot := filepath.Join(goenv.Get("GOCACHE"), cachedGorootName)
	if _, err := os.Stat(cachedgoroot); err == nil {
		return cachedgoroot, nil
	}

	// Create the cache directory if it does not already exist.
	err = os.MkdirAll(goenv.Get("GOCACHE"), 0777)
	if err != nil {
		return "", err
	}

	// Create a temporary directory to construct the goroot within.
	tmpgoroot, err := os.MkdirTemp(goenv.Get("GOCACHE"), cachedGorootName+".tmp")
	if err != nil {
		return "", err
	}

	// Remove the temporary directory if it wasn't moved to the right place
	// (for example, when there was an error).
	defer os.RemoveAll(tmpgoroot)

	// Create the directory structure.
	// The directories are created in sorted order so that nested directories are created without extra work.
	{
		var dirs []string
		for dir, merge := range overrides {
			if merge {
				dirs = append(dirs, filepath.Join(tmpgoroot, "src", dir))
			}
		}
		sort.Strings(dirs)

		for _, dir := range dirs {
			err := os.Mkdir(dir, 0777)
			if err != nil {
				return "", err
			}
		}
	}

	// Create all symlinks.
	for dst, src := range merge {
		err := symlink(src, filepath.Join(tmpgoroot, dst))
		if err != nil {
			return "", err
		}
	}

	// Rename the new merged gorooot into place.
	err = os.Rename(tmpgoroot, cachedgoroot)
	if err != nil {
		if errors.Is(err, fs.ErrExist) {
			// Another invocation of TinyGo also seems to have created a GOROOT.
			// Use that one instead. Our new GOROOT will be automatically
			// deleted by the defer above.
			return cachedgoroot, nil
		}
		if runtime.GOOS == "windows" && errors.Is(err, fs.ErrPermission) {
			// On Windows, a rename with a destination directory that already
			// exists does not result in an IsExist error, but rather in an
			// access denied error. To be sure, check for this case by checking
			// whether the target directory exists.
			if _, err := os.Stat(cachedgoroot); err == nil {
				return cachedgoroot, nil
			}
		}
		return "", err
	}
	return cachedgoroot, nil
}

// listGorootMergeLinks searches goroot and tinygoroot for all symlinks that must be created within the merged goroot.
func listGorootMergeLinks(goroot, tinygoroot string, overrides map[string]bool) (map[string]string, error) {
	goSrc := filepath.Join(goroot, "src")
	tinygoSrc := filepath.Join(tinygoroot, "src")
	merges := make(map[string]string)
	for dir, merge := range overrides {
		if !merge {
			// Use the TinyGo version.
			merges[filepath.Join("src", dir)] = filepath.Join(tinygoSrc, dir)
			continue
		}

		// Add files from TinyGo.
		tinygoDir := filepath.Join(tinygoSrc, dir)
		tinygoEntries, err := os.ReadDir(tinygoDir)
		if err != nil {
			return nil, err
		}
		var hasTinyGoFiles bool
		for _, e := range tinygoEntries {
			if e.IsDir() {
				continue
			}

			// Link this file.
			name := e.Name()
			merges[filepath.Join("src", dir, name)] = filepath.Join(tinygoDir, name)

			hasTinyGoFiles = true
		}

		// Add all directories from $GOROOT that are not part of the TinyGo
		// overrides.
		goDir := filepath.Join(goSrc, dir)
		goEntries, err := os.ReadDir(goDir)
		if err != nil {
			return nil, err
		}
		for _, e := range goEntries {
			isDir := e.IsDir()
			if hasTinyGoFiles && !isDir {
				// Only merge files from Go if TinyGo does not have any files.
				// Otherwise we'd end up with a weird mix from both Go
				// implementations.
				continue
			}

			name := e.Name()
			if _, ok := overrides[path.Join(dir, name)+"/"]; ok {
				// This entry is overridden by TinyGo.
				// It has/will be merged elsewhere.
				continue
			}

			// Add a link to this entry
			merges[filepath.Join("src", dir, name)] = filepath.Join(goDir, name)
		}
	}

	// Merge the special directories from goroot.
	for _, dir := range []string{"bin", "lib", "pkg"} {
		merges[dir] = filepath.Join(goroot, dir)
	}

	// Required starting in Go 1.21 due to https://github.com/golang/go/issues/61928
	if _, err := os.Stat(filepath.Join(goroot, "go.env")); err == nil {
		merges["go.env"] = filepath.Join(goroot, "go.env")
	}

	return merges, nil
}

// needsSyscallPackage returns whether the syscall package should be overridden
// with the TinyGo version. This is the case on some targets.
func needsSyscallPackage(buildTags []string) bool {
	for _, tag := range buildTags {
		if tag == "baremetal" || tag == "nintendoswitch" || tag == "tinygo.wasm" {
			return true
		}
	}
	return false
}

// The boolean indicates whether to merge the subdirs. True means merge, false
// means use the TinyGo version.
func pathsToOverride(goMinor int, needsSyscallPackage bool) map[string]bool {
	paths := map[string]bool{
		"":                            true,
		"crypto/":                     true,
		"crypto/rand/":                false,
		"crypto/tls/":                 false,
		"crypto/x509/":                true,
		"crypto/x509/internal/":       true,
		"crypto/x509/internal/macos/": false,
		"device/":                     false,
		"examples/":                   false,
		"internal/":                   true,
		"internal/abi/":               false,
		"internal/binary/":            false,
		"internal/bytealg/":           false,
		"internal/cm/":                false,
		"internal/futex/":             false,
		"internal/fuzz/":              false,
		"internal/reflectlite/":       false,
		"internal/gclayout":           false,
		"internal/task/":              false,
		"internal/wasi/":              false,
		"machine/":                    false,
		"net/":                        true,
		"net/http/":                   false,
		"os/":                         true,
		"reflect/":                    false,
		"runtime/":                    false,
		"sync/":                       true,
		"testing/":                    true,
		"tinygo/":                     false,
		"unique/":                     false,
	}

	if goMinor >= 19 {
		paths["crypto/internal/"] = true
		paths["crypto/internal/boring/"] = true
		paths["crypto/internal/boring/sig/"] = false
	}

	if needsSyscallPackage {
		paths["syscall/"] = true // include syscall/js
	}
	return paths
}

// symlink creates a symlink or something similar. On Unix-like systems, it
// always creates a symlink. On Windows, it tries to create a symlink and if
// that fails, creates a hardlink or directory junction instead.
//
// Note that while Windows 10 does support symlinks and allows them to be
// created using os.Symlink, it requires developer mode to be enabled.
// Therefore provide a fallback for when symlinking is not possible.
// Unfortunately this fallback only works when TinyGo is installed on the same
// filesystem as the TinyGo cache and the Go installation (which is usually the
// C drive).
func symlink(oldname, newname string) error {
	symlinkErr := os.Symlink(oldname, newname)
	if runtime.GOOS == "windows" && symlinkErr != nil {
		// Fallback for when developer mode is disabled.
		// Note that we return the symlink error even if something else fails
		// later on. This is because symlinks are the easiest to support
		// (they're also used on Linux and MacOS) and enabling them is easy:
		// just enable developer mode.
		st, err := os.Stat(oldname)
		if err != nil {
			return symlinkErr
		}
		if st.IsDir() {
			// Make a directory junction. There may be a way to do this
			// programmatically, but it involves a lot of magic. Use the mklink
			// command built into cmd instead (mklink is a builtin, not an
			// external command).
			err := exec.Command("cmd", "/k", "mklink", "/J", newname, oldname).Run()
			if err != nil {
				return symlinkErr
			}
		} else {
			// Try making a hard link.
			err := os.Link(oldname, newname)
			if err != nil {
				// Making a hardlink failed. Try copying the file as a last
				// fallback.
				inf, err := os.Open(oldname)
				if err != nil {
					return err
				}
				defer inf.Close()
				outf, err := os.Create(newname)
				if err != nil {
					return err
				}
				defer outf.Close()
				_, err = io.Copy(outf, inf)
				if err != nil {
					os.Remove(newname)
					return err
				}
				// File was copied.
			}
		}
		return nil // success
	}
	return symlinkErr
}